dat
commited on
Commit
•
0609cb0
1
Parent(s):
b681451
add all
Browse files- events.out.tfevents.1626318666.t1v-n-f5c06ea1-w-0.655476.3.v2 +2 -2
- events.out.tfevents.1626355795.t1v-n-f5c06ea1-w-0.688983.3.v2 +3 -0
- events.out.tfevents.1626357208.t1v-n-f5c06ea1-w-0.692253.3.v2 +3 -0
- run.sh +1 -1
- run_mlm_flax_no_accum.py +16 -15
- wandb/debug-internal.log +1 -1
- wandb/debug.log +1 -1
- wandb/latest-run +1 -1
- wandb/run-20210715_031107-69jkygz3/files/output.log +30 -0
- wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +1 -1
- wandb/run-20210715_031107-69jkygz3/logs/debug-internal.log +52 -0
- wandb/run-20210715_031107-69jkygz3/logs/debug.log +2 -0
- wandb/run-20210715_031107-69jkygz3/run-69jkygz3.wandb +0 -0
- wandb/run-20210715_132956-37l0zsn1/files/config.yaml +301 -0
- wandb/run-20210715_132956-37l0zsn1/files/output.log +195 -0
- wandb/run-20210715_132956-37l0zsn1/files/requirements.txt +94 -0
- wandb/run-20210715_132956-37l0zsn1/files/wandb-metadata.json +45 -0
- wandb/run-20210715_132956-37l0zsn1/files/wandb-summary.json +1 -0
- wandb/run-20210715_132956-37l0zsn1/logs/debug-internal.log +391 -0
- wandb/run-20210715_132956-37l0zsn1/logs/debug.log +27 -0
- wandb/run-20210715_132956-37l0zsn1/run-37l0zsn1.wandb +0 -0
- wandb/run-20210715_135329-1c6qs9ww/files/config.yaml +301 -0
- wandb/run-20210715_135329-1c6qs9ww/files/output.log +119 -0
- wandb/run-20210715_135329-1c6qs9ww/files/requirements.txt +94 -0
- wandb/run-20210715_135329-1c6qs9ww/files/wandb-metadata.json +45 -0
- wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json +1 -0
- wandb/run-20210715_135329-1c6qs9ww/logs/debug-internal.log +399 -0
- wandb/run-20210715_135329-1c6qs9ww/logs/debug.log +27 -0
- wandb/run-20210715_135329-1c6qs9ww/run-1c6qs9ww.wandb +0 -0
- wandb/run-20210715_141143-1vry5yso/files/config.yaml +301 -0
- wandb/run-20210715_141143-1vry5yso/files/output.log +64 -0
- wandb/run-20210715_141143-1vry5yso/files/requirements.txt +94 -0
- wandb/run-20210715_141143-1vry5yso/files/wandb-metadata.json +45 -0
- wandb/run-20210715_141143-1vry5yso/files/wandb-summary.json +1 -0
- wandb/run-20210715_141143-1vry5yso/logs/debug-internal.log +125 -0
- wandb/run-20210715_141143-1vry5yso/logs/debug.log +28 -0
- wandb/run-20210715_141143-1vry5yso/run-1vry5yso.wandb +0 -0
- wandb/run-20210715_142215-1vry5yso/files/config.yaml +20 -0
- wandb/run-20210715_142215-1vry5yso/logs/debug-internal.log +46 -0
- wandb/run-20210715_142215-1vry5yso/logs/debug.log +37 -0
- wandb/run-20210715_142215-1vry5yso/run-1vry5yso.wandb +0 -0
- wandb/wandb-resume.json +1 -0
events.out.tfevents.1626318666.t1v-n-f5c06ea1-w-0.655476.3.v2
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:898bcab2cec674ad70a77673aa240b8be9924c739c58fa92026ccb9953008588
|
3 |
+
size 13518440
|
events.out.tfevents.1626355795.t1v-n-f5c06ea1-w-0.688983.3.v2
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:b033e6ba5a9ecd725a4c22f2209304ae9f67b9f477543a2abd8c70b68fe4cc04
|
3 |
+
size 40
|
events.out.tfevents.1626357208.t1v-n-f5c06ea1-w-0.692253.3.v2
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:03fa64d5ad6dbc89223988a35e988bf0d38d9e67dc8d95fd1095b990ffc23a3e
|
3 |
+
size 148305
|
run.sh
CHANGED
@@ -15,7 +15,7 @@ python ./run_mlm_flax_no_accum.py \
|
|
15 |
--adam_beta1="0.9" \
|
16 |
--adam_beta2="0.98" \
|
17 |
--logging_steps="50" \
|
18 |
-
--eval_steps="
|
19 |
--num_train_epochs="2"\
|
20 |
--preprocessing_num_workers="96" \
|
21 |
--save_steps="10000" \
|
|
|
15 |
--adam_beta1="0.9" \
|
16 |
--adam_beta2="0.98" \
|
17 |
--logging_steps="50" \
|
18 |
+
--eval_steps="2000" \
|
19 |
--num_train_epochs="2"\
|
20 |
--preprocessing_num_workers="96" \
|
21 |
--save_steps="10000" \
|
run_mlm_flax_no_accum.py
CHANGED
@@ -59,6 +59,7 @@ from flax.training import checkpoints
|
|
59 |
from flax.jax_utils import unreplicate
|
60 |
from flax.training.checkpoints import save_checkpoint, restore_checkpoint
|
61 |
from importlib.util import find_spec
|
|
|
62 |
|
63 |
|
64 |
MODEL_CONFIG_CLASSES = list(FLAX_MODEL_FOR_MASKED_LM_MAPPING.keys())
|
@@ -270,16 +271,16 @@ def write_eval_metric(summary_writer, eval_metrics, step):
|
|
270 |
for metric_name, value in eval_metrics.items():
|
271 |
summary_writer.scalar(f"eval_{metric_name}", value, step)
|
272 |
|
273 |
-
def rotate_checkpoints(ckpt_dir:str, save_total_limit:int):
|
274 |
-
"Removes older checkpoints so that `save_total_limit` checkpoints are kept"
|
275 |
-
# TODO: what to remove is decided using step number only, we might want to improve that
|
276 |
-
ckpts = [str(x) for x in Path(ckpt_dir).glob("ckpt-*")]
|
277 |
-
# sort checkpoints by step
|
278 |
-
ckpts_sorted = sorted(ckpts, key=lambda x: int(x.split('-')[-1]))
|
279 |
-
ckpts_to_delete = ckpts_sorted[:-save_total_limit]
|
280 |
-
for ckpt in ckpts_to_delete:
|
281 |
-
logger.info(f"Deleting older checkpoint [{ckpt}] due to save_total_limit ({save_total_limit})")
|
282 |
-
shutil.rmtree(ckpt)
|
283 |
|
284 |
|
285 |
if __name__ == "__main__":
|
@@ -530,7 +531,7 @@ if __name__ == "__main__":
|
|
530 |
wandb.init(
|
531 |
entity="wandb",
|
532 |
project="hf-flax-pino-roberta",
|
533 |
-
sync_tensorboard=True
|
534 |
)
|
535 |
wandb.config.update(training_args)
|
536 |
wandb.config.update(model_args)
|
@@ -700,8 +701,8 @@ if __name__ == "__main__":
|
|
700 |
train_metrics.append(train_metric)
|
701 |
|
702 |
cur_step = epoch * (num_train_samples // train_batch_size) + step
|
703 |
-
if cur_step < resume_step:
|
704 |
-
|
705 |
|
706 |
if cur_step % training_args.logging_steps == 0 and cur_step > 0:
|
707 |
# Save metrics
|
@@ -764,8 +765,8 @@ if __name__ == "__main__":
|
|
764 |
push_to_hub=training_args.push_to_hub,
|
765 |
commit_message=f"Saving weights and logs of step {cur_step}",
|
766 |
)
|
767 |
-
if training_args.save_total_limit is not None:
|
768 |
-
|
769 |
if jax.process_index() == 0:
|
770 |
params = jax.device_get(jax.tree_map(lambda x: x[0], state.params))
|
771 |
model.save_pretrained(
|
|
|
59 |
from flax.jax_utils import unreplicate
|
60 |
from flax.training.checkpoints import save_checkpoint, restore_checkpoint
|
61 |
from importlib.util import find_spec
|
62 |
+
import shutil
|
63 |
|
64 |
|
65 |
MODEL_CONFIG_CLASSES = list(FLAX_MODEL_FOR_MASKED_LM_MAPPING.keys())
|
|
|
271 |
for metric_name, value in eval_metrics.items():
|
272 |
summary_writer.scalar(f"eval_{metric_name}", value, step)
|
273 |
|
274 |
+
#def rotate_checkpoints(ckpt_dir:str, save_total_limit:int):
|
275 |
+
# "Removes older checkpoints so that `save_total_limit` checkpoints are kept"
|
276 |
+
# # TODO: what to remove is decided using step number only, we might want to improve that
|
277 |
+
# ckpts = [str(x) for x in Path(ckpt_dir).glob("ckpt-*")]
|
278 |
+
# # sort checkpoints by step
|
279 |
+
# ckpts_sorted = sorted(ckpts, key=lambda x: int(x.split('-')[-1]))
|
280 |
+
# ckpts_to_delete = ckpts_sorted[:-save_total_limit]
|
281 |
+
# for ckpt in ckpts_to_delete:
|
282 |
+
# logger.info(f"Deleting older checkpoint [{ckpt}] due to save_total_limit ({save_total_limit})")
|
283 |
+
# shutil.rmtree(ckpt)
|
284 |
|
285 |
|
286 |
if __name__ == "__main__":
|
|
|
531 |
wandb.init(
|
532 |
entity="wandb",
|
533 |
project="hf-flax-pino-roberta",
|
534 |
+
sync_tensorboard=True,
|
535 |
)
|
536 |
wandb.config.update(training_args)
|
537 |
wandb.config.update(model_args)
|
|
|
701 |
train_metrics.append(train_metric)
|
702 |
|
703 |
cur_step = epoch * (num_train_samples // train_batch_size) + step
|
704 |
+
#if cur_step < resume_step:
|
705 |
+
# continue
|
706 |
|
707 |
if cur_step % training_args.logging_steps == 0 and cur_step > 0:
|
708 |
# Save metrics
|
|
|
765 |
push_to_hub=training_args.push_to_hub,
|
766 |
commit_message=f"Saving weights and logs of step {cur_step}",
|
767 |
)
|
768 |
+
#if training_args.save_total_limit is not None:
|
769 |
+
# rotate_checkpoints(training_args.output_dir, training_args.save_total_limit)
|
770 |
if jax.process_index() == 0:
|
771 |
params = jax.device_get(jax.tree_map(lambda x: x[0], state.params))
|
772 |
model.save_pretrained(
|
wandb/debug-internal.log
CHANGED
@@ -1 +1 @@
|
|
1 |
-
run-
|
|
|
1 |
+
run-20210715_142215-1vry5yso/logs/debug-internal.log
|
wandb/debug.log
CHANGED
@@ -1 +1 @@
|
|
1 |
-
run-
|
|
|
1 |
+
run-20210715_142215-1vry5yso/logs/debug.log
|
wandb/latest-run
CHANGED
@@ -1 +1 @@
|
|
1 |
-
run-
|
|
|
1 |
+
run-20210715_142215-1vry5yso
|
wandb/run-20210715_031107-69jkygz3/files/output.log
CHANGED
@@ -10534,3 +10534,33 @@ tcmalloc: large alloc 2715181056 bytes == 0x2d6562000 @ 0x7f4e39cd4680 0x7f4e39
|
|
10534 |
[13:26:07] - INFO - absl - Saved checkpoint at checkpoint_90000
|
10535 |
[13:26:09] - INFO - huggingface_hub.repository - git version 2.25.1
|
10536 |
git-lfs/2.9.2 (GitHub; linux amd64; go 1.13.5)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
10534 |
[13:26:07] - INFO - absl - Saved checkpoint at checkpoint_90000
|
10535 |
[13:26:09] - INFO - huggingface_hub.repository - git version 2.25.1
|
10536 |
git-lfs/2.9.2 (GitHub; linux amd64; go 1.13.5)
|
10537 |
+
[13:26:09] - DEBUG - huggingface_hub.repository - [Repository] is a valid git repo
|
10538 |
+
[13:27:29] - INFO - huggingface_hub.repository - Uploading LFS objects: 100% (3/3), 2.1 GB | 43 MB/s, done.
|
10539 |
+
|
10540 |
+
|
10541 |
+
|
10542 |
+
Training...: 26%|██████████████▊ | 90050/352766 [10:10:46<26:20:31, 2.77it/s]
|
10543 |
+
|
10544 |
+
|
10545 |
+
|
10546 |
+
|
10547 |
+
Training...: 26%|██████████████▊ | 90100/352766 [10:11:06<26:08:06, 2.79it/s]
|
10548 |
+
|
10549 |
+
Step... (80000 | Loss: 2.5656967163085938, Acc: 0.533167839050293): 0%| | 0/5 [10:16:51<?, ?it/s]
|
10550 |
+
Traceback (most recent call last):
|
10551 |
+
File "./run_mlm_flax_no_accum.py", line 694, in <module>
|
10552 |
+
samples = [tokenized_datasets["train"][int(idx)] for idx in batch_idx]
|
10553 |
+
File "./run_mlm_flax_no_accum.py", line 694, in <listcomp>
|
10554 |
+
samples = [tokenized_datasets["train"][int(idx)] for idx in batch_idx]
|
10555 |
+
File "/home/dat/datasets/src/datasets/arrow_dataset.py", line 1517, in __getitem__
|
10556 |
+
return self._getitem(
|
10557 |
+
File "/home/dat/datasets/src/datasets/arrow_dataset.py", line 1510, in _getitem
|
10558 |
+
formatted_output = format_table(
|
10559 |
+
File "/home/dat/datasets/src/datasets/formatting/formatting.py", line 414, in format_table
|
10560 |
+
return formatter(pa_table, query_type=query_type)
|
10561 |
+
File "/home/dat/datasets/src/datasets/formatting/formatting.py", line 194, in __call__
|
10562 |
+
return self.format_row(pa_table)
|
10563 |
+
File "/home/dat/datasets/src/datasets/formatting/formatting.py", line 223, in format_row
|
10564 |
+
return self.python_arrow_extractor().extract_row(pa_table)
|
10565 |
+
File "/home/dat/datasets/src/datasets/formatting/formatting.py", line 134, in extract_row
|
10566 |
+
return _unnest(pa_table.to_pydict())
|
wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json
CHANGED
@@ -1 +1 @@
|
|
1 |
-
{"training_step":
|
|
|
1 |
+
{"training_step": 90100, "learning_rate": 2.8629854568862356e-05, "train_loss": 2.7210891246795654, "_runtime": 37024, "_timestamp": 1626355691, "_step": 1805, "eval_step": 80000, "eval_accuracy": 0.533167839050293, "eval_loss": 2.5656967163085938}
|
wandb/run-20210715_031107-69jkygz3/logs/debug-internal.log
CHANGED
@@ -23072,3 +23072,55 @@
|
|
23072 |
2021-07-15 13:26:12,804 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log
|
23073 |
2021-07-15 13:26:26,096 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status
|
23074 |
2021-07-15 13:26:26,097 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
23072 |
2021-07-15 13:26:12,804 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log
|
23073 |
2021-07-15 13:26:26,096 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status
|
23074 |
2021-07-15 13:26:26,097 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status
|
23075 |
+
2021-07-15 13:26:41,229 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status
|
23076 |
+
2021-07-15 13:26:41,229 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status
|
23077 |
+
2021-07-15 13:26:42,234 DEBUG SenderThread:656728 [sender.py:send():179] send: stats
|
23078 |
+
2021-07-15 13:26:56,477 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status
|
23079 |
+
2021-07-15 13:26:56,478 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status
|
23080 |
+
2021-07-15 13:27:11,612 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status
|
23081 |
+
2021-07-15 13:27:11,612 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status
|
23082 |
+
2021-07-15 13:27:12,309 DEBUG SenderThread:656728 [sender.py:send():179] send: stats
|
23083 |
+
2021-07-15 13:27:26,746 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status
|
23084 |
+
2021-07-15 13:27:26,747 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status
|
23085 |
+
2021-07-15 13:27:32,838 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log
|
23086 |
+
2021-07-15 13:27:34,839 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log
|
23087 |
+
2021-07-15 13:27:36,840 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log
|
23088 |
+
2021-07-15 13:27:38,842 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log
|
23089 |
+
2021-07-15 13:27:41,877 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status
|
23090 |
+
2021-07-15 13:27:41,878 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status
|
23091 |
+
2021-07-15 13:27:42,386 DEBUG SenderThread:656728 [sender.py:send():179] send: stats
|
23092 |
+
2021-07-15 13:27:50,988 DEBUG SenderThread:656728 [sender.py:send():179] send: history
|
23093 |
+
2021-07-15 13:27:50,988 DEBUG SenderThread:656728 [sender.py:send():179] send: summary
|
23094 |
+
2021-07-15 13:27:50,989 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
|
23095 |
+
2021-07-15 13:27:51,847 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json
|
23096 |
+
2021-07-15 13:27:52,848 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log
|
23097 |
+
2021-07-15 13:27:54,849 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log
|
23098 |
+
2021-07-15 13:27:56,850 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log
|
23099 |
+
2021-07-15 13:27:57,009 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status
|
23100 |
+
2021-07-15 13:27:57,009 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status
|
23101 |
+
2021-07-15 13:27:58,851 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log
|
23102 |
+
2021-07-15 13:28:11,019 DEBUG SenderThread:656728 [sender.py:send():179] send: history
|
23103 |
+
2021-07-15 13:28:11,019 DEBUG SenderThread:656728 [sender.py:send():179] send: summary
|
23104 |
+
2021-07-15 13:28:11,019 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
|
23105 |
+
2021-07-15 13:28:11,857 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json
|
23106 |
+
2021-07-15 13:28:12,168 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status
|
23107 |
+
2021-07-15 13:28:12,169 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status
|
23108 |
+
2021-07-15 13:28:12,464 DEBUG SenderThread:656728 [sender.py:send():179] send: stats
|
23109 |
+
2021-07-15 13:28:12,857 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log
|
23110 |
+
2021-07-15 13:28:13,729 WARNING MainThread:656728 [internal.py:wandb_internal():147] Internal process interrupt: 1
|
23111 |
+
2021-07-15 13:28:14,628 WARNING MainThread:656728 [internal.py:wandb_internal():147] Internal process interrupt: 2
|
23112 |
+
2021-07-15 13:28:14,628 ERROR MainThread:656728 [internal.py:wandb_internal():150] Internal process interrupted.
|
23113 |
+
2021-07-15 13:28:14,858 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log
|
23114 |
+
2021-07-15 13:28:15,239 INFO SenderThread:656728 [sender.py:finish():945] shutting down sender
|
23115 |
+
2021-07-15 13:28:15,240 INFO WriterThread:656728 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/run-69jkygz3.wandb
|
23116 |
+
2021-07-15 13:28:15,240 INFO SenderThread:656728 [dir_watcher.py:finish():282] shutting down directory watcher
|
23117 |
+
2021-07-15 13:28:15,240 INFO HandlerThread:656728 [handler.py:finish():638] shutting down handler
|
23118 |
+
2021-07-15 13:28:15,859 INFO SenderThread:656728 [dir_watcher.py:finish():312] scan: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files
|
23119 |
+
2021-07-15 13:28:15,860 INFO SenderThread:656728 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/requirements.txt requirements.txt
|
23120 |
+
2021-07-15 13:28:15,860 INFO SenderThread:656728 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log output.log
|
23121 |
+
2021-07-15 13:28:15,860 INFO SenderThread:656728 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-metadata.json wandb-metadata.json
|
23122 |
+
2021-07-15 13:28:15,860 INFO SenderThread:656728 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/config.yaml config.yaml
|
23123 |
+
2021-07-15 13:28:15,860 INFO SenderThread:656728 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json wandb-summary.json
|
23124 |
+
2021-07-15 13:28:15,861 INFO SenderThread:656728 [file_pusher.py:finish():177] shutting down file pusher
|
23125 |
+
2021-07-15 13:28:15,861 INFO SenderThread:656728 [file_pusher.py:join():182] waiting for file pusher
|
23126 |
+
2021-07-15 13:28:16,239 INFO MainThread:656728 [internal.py:handle_exit():78] Internal process exited
|
wandb/run-20210715_031107-69jkygz3/logs/debug.log
CHANGED
@@ -23,3 +23,5 @@ config: {}
|
|
23 |
2021-07-15 03:11:09,649 INFO MainThread:655476 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 3e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul15_03-10-59_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 50, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 30000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 20000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''}
|
24 |
2021-07-15 03:11:09,651 INFO MainThread:655476 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'}
|
25 |
2021-07-15 03:11:09,653 INFO MainThread:655476 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 500}
|
|
|
|
|
|
23 |
2021-07-15 03:11:09,649 INFO MainThread:655476 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 3e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul15_03-10-59_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 50, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 30000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 20000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''}
|
24 |
2021-07-15 03:11:09,651 INFO MainThread:655476 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'}
|
25 |
2021-07-15 03:11:09,653 INFO MainThread:655476 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 500}
|
26 |
+
2021-07-15 13:28:13,756 INFO MainThread:655476 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 255
|
27 |
+
2021-07-15 13:28:13,756 INFO MainThread:655476 [wandb_run.py:_restore():1565] restore
|
wandb/run-20210715_031107-69jkygz3/run-69jkygz3.wandb
CHANGED
Binary files a/wandb/run-20210715_031107-69jkygz3/run-69jkygz3.wandb and b/wandb/run-20210715_031107-69jkygz3/run-69jkygz3.wandb differ
|
|
wandb/run-20210715_132956-37l0zsn1/files/config.yaml
ADDED
@@ -0,0 +1,301 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
wandb_version: 1
|
2 |
+
|
3 |
+
_wandb:
|
4 |
+
desc: null
|
5 |
+
value:
|
6 |
+
cli_version: 0.10.33
|
7 |
+
framework: huggingface
|
8 |
+
huggingface_version: 4.9.0.dev0
|
9 |
+
is_jupyter_run: false
|
10 |
+
is_kaggle_kernel: false
|
11 |
+
python_version: 3.8.10
|
12 |
+
t:
|
13 |
+
1:
|
14 |
+
- 3
|
15 |
+
- 11
|
16 |
+
4: 3.8.10
|
17 |
+
5: 0.10.33
|
18 |
+
6: 4.9.0.dev0
|
19 |
+
8:
|
20 |
+
- 5
|
21 |
+
adafactor:
|
22 |
+
desc: null
|
23 |
+
value: false
|
24 |
+
adam_beta1:
|
25 |
+
desc: null
|
26 |
+
value: 0.9
|
27 |
+
adam_beta2:
|
28 |
+
desc: null
|
29 |
+
value: 0.98
|
30 |
+
adam_epsilon:
|
31 |
+
desc: null
|
32 |
+
value: 1.0e-08
|
33 |
+
cache_dir:
|
34 |
+
desc: null
|
35 |
+
value: null
|
36 |
+
config_name:
|
37 |
+
desc: null
|
38 |
+
value: ./
|
39 |
+
dataloader_drop_last:
|
40 |
+
desc: null
|
41 |
+
value: false
|
42 |
+
dataloader_num_workers:
|
43 |
+
desc: null
|
44 |
+
value: 0
|
45 |
+
dataloader_pin_memory:
|
46 |
+
desc: null
|
47 |
+
value: true
|
48 |
+
dataset_config_name:
|
49 |
+
desc: null
|
50 |
+
value: null
|
51 |
+
dataset_name:
|
52 |
+
desc: null
|
53 |
+
value: null
|
54 |
+
ddp_find_unused_parameters:
|
55 |
+
desc: null
|
56 |
+
value: null
|
57 |
+
debug:
|
58 |
+
desc: null
|
59 |
+
value: []
|
60 |
+
deepspeed:
|
61 |
+
desc: null
|
62 |
+
value: null
|
63 |
+
disable_tqdm:
|
64 |
+
desc: null
|
65 |
+
value: false
|
66 |
+
do_eval:
|
67 |
+
desc: null
|
68 |
+
value: false
|
69 |
+
do_predict:
|
70 |
+
desc: null
|
71 |
+
value: false
|
72 |
+
do_train:
|
73 |
+
desc: null
|
74 |
+
value: false
|
75 |
+
dtype:
|
76 |
+
desc: null
|
77 |
+
value: float32
|
78 |
+
eval_accumulation_steps:
|
79 |
+
desc: null
|
80 |
+
value: null
|
81 |
+
eval_steps:
|
82 |
+
desc: null
|
83 |
+
value: 2000
|
84 |
+
evaluation_strategy:
|
85 |
+
desc: null
|
86 |
+
value: IntervalStrategy.NO
|
87 |
+
fp16:
|
88 |
+
desc: null
|
89 |
+
value: false
|
90 |
+
fp16_backend:
|
91 |
+
desc: null
|
92 |
+
value: auto
|
93 |
+
fp16_full_eval:
|
94 |
+
desc: null
|
95 |
+
value: false
|
96 |
+
fp16_opt_level:
|
97 |
+
desc: null
|
98 |
+
value: O1
|
99 |
+
gradient_accumulation_steps:
|
100 |
+
desc: null
|
101 |
+
value: 1
|
102 |
+
greater_is_better:
|
103 |
+
desc: null
|
104 |
+
value: null
|
105 |
+
group_by_length:
|
106 |
+
desc: null
|
107 |
+
value: false
|
108 |
+
ignore_data_skip:
|
109 |
+
desc: null
|
110 |
+
value: false
|
111 |
+
label_names:
|
112 |
+
desc: null
|
113 |
+
value: null
|
114 |
+
label_smoothing_factor:
|
115 |
+
desc: null
|
116 |
+
value: 0.0
|
117 |
+
learning_rate:
|
118 |
+
desc: null
|
119 |
+
value: 3.0e-05
|
120 |
+
length_column_name:
|
121 |
+
desc: null
|
122 |
+
value: length
|
123 |
+
line_by_line:
|
124 |
+
desc: null
|
125 |
+
value: false
|
126 |
+
load_best_model_at_end:
|
127 |
+
desc: null
|
128 |
+
value: false
|
129 |
+
local_rank:
|
130 |
+
desc: null
|
131 |
+
value: -1
|
132 |
+
log_level:
|
133 |
+
desc: null
|
134 |
+
value: -1
|
135 |
+
log_level_replica:
|
136 |
+
desc: null
|
137 |
+
value: -1
|
138 |
+
log_on_each_node:
|
139 |
+
desc: null
|
140 |
+
value: true
|
141 |
+
logging_dir:
|
142 |
+
desc: null
|
143 |
+
value: ./runs/Jul15_13-29-47_t1v-n-f5c06ea1-w-0
|
144 |
+
logging_first_step:
|
145 |
+
desc: null
|
146 |
+
value: false
|
147 |
+
logging_steps:
|
148 |
+
desc: null
|
149 |
+
value: 50
|
150 |
+
logging_strategy:
|
151 |
+
desc: null
|
152 |
+
value: IntervalStrategy.STEPS
|
153 |
+
lr_scheduler_type:
|
154 |
+
desc: null
|
155 |
+
value: SchedulerType.LINEAR
|
156 |
+
max_eval_samples:
|
157 |
+
desc: null
|
158 |
+
value: 5000
|
159 |
+
max_grad_norm:
|
160 |
+
desc: null
|
161 |
+
value: 1.0
|
162 |
+
max_seq_length:
|
163 |
+
desc: null
|
164 |
+
value: 4096
|
165 |
+
max_steps:
|
166 |
+
desc: null
|
167 |
+
value: -1
|
168 |
+
metric_for_best_model:
|
169 |
+
desc: null
|
170 |
+
value: null
|
171 |
+
mlm_probability:
|
172 |
+
desc: null
|
173 |
+
value: 0.15
|
174 |
+
model_name_or_path:
|
175 |
+
desc: null
|
176 |
+
value: null
|
177 |
+
model_type:
|
178 |
+
desc: null
|
179 |
+
value: big_bird
|
180 |
+
mp_parameters:
|
181 |
+
desc: null
|
182 |
+
value: ''
|
183 |
+
no_cuda:
|
184 |
+
desc: null
|
185 |
+
value: false
|
186 |
+
num_train_epochs:
|
187 |
+
desc: null
|
188 |
+
value: 2.0
|
189 |
+
output_dir:
|
190 |
+
desc: null
|
191 |
+
value: ./
|
192 |
+
overwrite_cache:
|
193 |
+
desc: null
|
194 |
+
value: false
|
195 |
+
overwrite_output_dir:
|
196 |
+
desc: null
|
197 |
+
value: true
|
198 |
+
pad_to_max_length:
|
199 |
+
desc: null
|
200 |
+
value: false
|
201 |
+
past_index:
|
202 |
+
desc: null
|
203 |
+
value: -1
|
204 |
+
per_device_eval_batch_size:
|
205 |
+
desc: null
|
206 |
+
value: 1
|
207 |
+
per_device_train_batch_size:
|
208 |
+
desc: null
|
209 |
+
value: 1
|
210 |
+
per_gpu_eval_batch_size:
|
211 |
+
desc: null
|
212 |
+
value: null
|
213 |
+
per_gpu_train_batch_size:
|
214 |
+
desc: null
|
215 |
+
value: null
|
216 |
+
prediction_loss_only:
|
217 |
+
desc: null
|
218 |
+
value: false
|
219 |
+
preprocessing_num_workers:
|
220 |
+
desc: null
|
221 |
+
value: 96
|
222 |
+
push_to_hub:
|
223 |
+
desc: null
|
224 |
+
value: true
|
225 |
+
push_to_hub_model_id:
|
226 |
+
desc: null
|
227 |
+
value: ''
|
228 |
+
push_to_hub_organization:
|
229 |
+
desc: null
|
230 |
+
value: null
|
231 |
+
push_to_hub_token:
|
232 |
+
desc: null
|
233 |
+
value: null
|
234 |
+
remove_unused_columns:
|
235 |
+
desc: null
|
236 |
+
value: true
|
237 |
+
report_to:
|
238 |
+
desc: null
|
239 |
+
value:
|
240 |
+
- tensorboard
|
241 |
+
- wandb
|
242 |
+
resume_from_checkpoint:
|
243 |
+
desc: null
|
244 |
+
value: ./
|
245 |
+
run_name:
|
246 |
+
desc: null
|
247 |
+
value: ./
|
248 |
+
save_on_each_node:
|
249 |
+
desc: null
|
250 |
+
value: false
|
251 |
+
save_steps:
|
252 |
+
desc: null
|
253 |
+
value: 10000
|
254 |
+
save_strategy:
|
255 |
+
desc: null
|
256 |
+
value: IntervalStrategy.STEPS
|
257 |
+
save_total_limit:
|
258 |
+
desc: null
|
259 |
+
value: 5
|
260 |
+
seed:
|
261 |
+
desc: null
|
262 |
+
value: 42
|
263 |
+
sharded_ddp:
|
264 |
+
desc: null
|
265 |
+
value: []
|
266 |
+
skip_memory_metrics:
|
267 |
+
desc: null
|
268 |
+
value: true
|
269 |
+
tokenizer_name:
|
270 |
+
desc: null
|
271 |
+
value: ./
|
272 |
+
tpu_metrics_debug:
|
273 |
+
desc: null
|
274 |
+
value: false
|
275 |
+
tpu_num_cores:
|
276 |
+
desc: null
|
277 |
+
value: null
|
278 |
+
train_ref_file:
|
279 |
+
desc: null
|
280 |
+
value: null
|
281 |
+
use_fast_tokenizer:
|
282 |
+
desc: null
|
283 |
+
value: true
|
284 |
+
use_legacy_prediction_loop:
|
285 |
+
desc: null
|
286 |
+
value: false
|
287 |
+
validation_ref_file:
|
288 |
+
desc: null
|
289 |
+
value: null
|
290 |
+
validation_split_percentage:
|
291 |
+
desc: null
|
292 |
+
value: 5
|
293 |
+
warmup_ratio:
|
294 |
+
desc: null
|
295 |
+
value: 0.0
|
296 |
+
warmup_steps:
|
297 |
+
desc: null
|
298 |
+
value: 10000
|
299 |
+
weight_decay:
|
300 |
+
desc: null
|
301 |
+
value: 0.0095
|
wandb/run-20210715_132956-37l0zsn1/files/output.log
ADDED
@@ -0,0 +1,195 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
[13:30:11] - INFO - absl - Restoring checkpoint from ./checkpoint_90000
|
2 |
+
tcmalloc: large alloc 1530273792 bytes == 0x9b1a4000 @ 0x7fd1962e2680 0x7fd196303824 0x5b9a14 0x50b2ae 0x50cb1b 0x5a6f17 0x5f3010 0x56fd36 0x568d9a 0x5f5b33 0x56aadf 0x568d9a 0x68cdc7 0x67e161 0x67e1df 0x67e281 0x67e627 0x6b6e62 0x6b71ed 0x7fd1960f70b3 0x5f96de
|
3 |
+
/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:386: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code.
|
4 |
+
warnings.warn(
|
5 |
+
/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:373: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code.
|
6 |
+
warnings.warn(
|
7 |
+
Epoch ... (1/2): 0%| | 0/2 [00:00<?, ?it/s]
|
8 |
+
|
9 |
+
|
10 |
+
|
11 |
+
|
12 |
+
|
13 |
+
|
14 |
+
|
15 |
+
|
16 |
+
|
17 |
+
|
18 |
+
|
19 |
+
|
20 |
+
|
21 |
+
|
22 |
+
|
23 |
+
|
24 |
+
|
25 |
+
|
26 |
+
|
27 |
+
|
28 |
+
|
29 |
+
|
30 |
+
|
31 |
+
|
32 |
+
|
33 |
+
|
34 |
+
|
35 |
+
|
36 |
+
|
37 |
+
|
38 |
+
|
39 |
+
|
40 |
+
|
41 |
+
|
42 |
+
|
43 |
+
|
44 |
+
|
45 |
+
|
46 |
+
|
47 |
+
|
48 |
+
|
49 |
+
|
50 |
+
|
51 |
+
|
52 |
+
|
53 |
+
|
54 |
+
|
55 |
+
|
56 |
+
|
57 |
+
|
58 |
+
|
59 |
+
|
60 |
+
|
61 |
+
|
62 |
+
|
63 |
+
|
64 |
+
|
65 |
+
|
66 |
+
|
67 |
+
|
68 |
+
|
69 |
+
|
70 |
+
|
71 |
+
|
72 |
+
|
73 |
+
|
74 |
+
|
75 |
+
|
76 |
+
|
77 |
+
|
78 |
+
|
79 |
+
|
80 |
+
|
81 |
+
|
82 |
+
|
83 |
+
|
84 |
+
|
85 |
+
|
86 |
+
|
87 |
+
|
88 |
+
|
89 |
+
|
90 |
+
|
91 |
+
|
92 |
+
|
93 |
+
|
94 |
+
|
95 |
+
|
96 |
+
|
97 |
+
|
98 |
+
|
99 |
+
|
100 |
+
|
101 |
+
|
102 |
+
|
103 |
+
|
104 |
+
|
105 |
+
|
106 |
+
|
107 |
+
|
108 |
+
|
109 |
+
|
110 |
+
|
111 |
+
|
112 |
+
|
113 |
+
|
114 |
+
|
115 |
+
|
116 |
+
|
117 |
+
|
118 |
+
|
119 |
+
|
120 |
+
|
121 |
+
|
122 |
+
|
123 |
+
|
124 |
+
|
125 |
+
|
126 |
+
|
127 |
+
|
128 |
+
|
129 |
+
|
130 |
+
|
131 |
+
|
132 |
+
|
133 |
+
|
134 |
+
|
135 |
+
|
136 |
+
|
137 |
+
|
138 |
+
|
139 |
+
|
140 |
+
|
141 |
+
|
142 |
+
|
143 |
+
|
144 |
+
|
145 |
+
|
146 |
+
|
147 |
+
|
148 |
+
|
149 |
+
|
150 |
+
|
151 |
+
|
152 |
+
|
153 |
+
|
154 |
+
|
155 |
+
|
156 |
+
|
157 |
+
|
158 |
+
|
159 |
+
|
160 |
+
|
161 |
+
|
162 |
+
|
163 |
+
|
164 |
+
|
165 |
+
|
166 |
+
|
167 |
+
|
168 |
+
|
169 |
+
|
170 |
+
|
171 |
+
|
172 |
+
|
173 |
+
|
174 |
+
|
175 |
+
|
176 |
+
|
177 |
+
|
178 |
+
|
179 |
+
|
180 |
+
|
181 |
+
|
182 |
+
|
183 |
+
|
184 |
+
|
185 |
+
|
186 |
+
|
187 |
+
|
188 |
+
|
189 |
+
|
190 |
+
|
191 |
+
|
192 |
+
|
193 |
+
|
194 |
+
|
195 |
+
|
wandb/run-20210715_132956-37l0zsn1/files/requirements.txt
ADDED
@@ -0,0 +1,94 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
absl-py==0.13.0
|
2 |
+
aiohttp==3.7.4.post0
|
3 |
+
astunparse==1.6.3
|
4 |
+
async-timeout==3.0.1
|
5 |
+
attrs==21.2.0
|
6 |
+
cachetools==4.2.2
|
7 |
+
certifi==2021.5.30
|
8 |
+
chardet==4.0.0
|
9 |
+
charset-normalizer==2.0.1
|
10 |
+
chex==0.0.8
|
11 |
+
click==8.0.1
|
12 |
+
configparser==5.0.2
|
13 |
+
cycler==0.10.0
|
14 |
+
datasets==1.9.1.dev0
|
15 |
+
dill==0.3.4
|
16 |
+
dm-tree==0.1.6
|
17 |
+
docker-pycreds==0.4.0
|
18 |
+
filelock==3.0.12
|
19 |
+
flatbuffers==1.12
|
20 |
+
flax==0.3.4
|
21 |
+
fsspec==2021.7.0
|
22 |
+
gast==0.4.0
|
23 |
+
gitdb==4.0.7
|
24 |
+
gitpython==3.1.18
|
25 |
+
google-auth-oauthlib==0.4.4
|
26 |
+
google-auth==1.32.1
|
27 |
+
google-pasta==0.2.0
|
28 |
+
grpcio==1.34.1
|
29 |
+
h5py==3.1.0
|
30 |
+
huggingface-hub==0.0.12
|
31 |
+
idna==3.2
|
32 |
+
install==1.3.4
|
33 |
+
jax==0.2.17
|
34 |
+
jaxlib==0.1.68
|
35 |
+
joblib==1.0.1
|
36 |
+
keras-nightly==2.5.0.dev2021032900
|
37 |
+
keras-preprocessing==1.1.2
|
38 |
+
kiwisolver==1.3.1
|
39 |
+
libtpu-nightly==0.1.dev20210615
|
40 |
+
markdown==3.3.4
|
41 |
+
matplotlib==3.4.2
|
42 |
+
msgpack==1.0.2
|
43 |
+
multidict==5.1.0
|
44 |
+
multiprocess==0.70.12.2
|
45 |
+
numpy==1.19.5
|
46 |
+
oauthlib==3.1.1
|
47 |
+
opt-einsum==3.3.0
|
48 |
+
optax==0.0.9
|
49 |
+
packaging==21.0
|
50 |
+
pandas==1.3.0
|
51 |
+
pathtools==0.1.2
|
52 |
+
pillow==8.3.1
|
53 |
+
pip==20.0.2
|
54 |
+
pkg-resources==0.0.0
|
55 |
+
promise==2.3
|
56 |
+
protobuf==3.17.3
|
57 |
+
psutil==5.8.0
|
58 |
+
pyarrow==4.0.1
|
59 |
+
pyasn1-modules==0.2.8
|
60 |
+
pyasn1==0.4.8
|
61 |
+
pyparsing==2.4.7
|
62 |
+
python-dateutil==2.8.1
|
63 |
+
pytz==2021.1
|
64 |
+
pyyaml==5.4.1
|
65 |
+
regex==2021.7.6
|
66 |
+
requests-oauthlib==1.3.0
|
67 |
+
requests==2.26.0
|
68 |
+
rsa==4.7.2
|
69 |
+
sacremoses==0.0.45
|
70 |
+
scipy==1.7.0
|
71 |
+
sentry-sdk==1.3.0
|
72 |
+
setuptools==44.0.0
|
73 |
+
shortuuid==1.0.1
|
74 |
+
six==1.15.0
|
75 |
+
smmap==4.0.0
|
76 |
+
subprocess32==3.5.4
|
77 |
+
tensorboard-data-server==0.6.1
|
78 |
+
tensorboard-plugin-wit==1.8.0
|
79 |
+
tensorboard==2.5.0
|
80 |
+
tensorflow-estimator==2.5.0
|
81 |
+
tensorflow==2.5.0
|
82 |
+
termcolor==1.1.0
|
83 |
+
tokenizers==0.10.3
|
84 |
+
toolz==0.11.1
|
85 |
+
tqdm==4.61.2
|
86 |
+
transformers==4.9.0.dev0
|
87 |
+
typing-extensions==3.7.4.3
|
88 |
+
urllib3==1.26.6
|
89 |
+
wandb==0.10.33
|
90 |
+
werkzeug==2.0.1
|
91 |
+
wheel==0.36.2
|
92 |
+
wrapt==1.12.1
|
93 |
+
xxhash==2.0.2
|
94 |
+
yarl==1.6.3
|
wandb/run-20210715_132956-37l0zsn1/files/wandb-metadata.json
ADDED
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29",
|
3 |
+
"python": "3.8.10",
|
4 |
+
"heartbeatAt": "2021-07-15T13:29:58.227318",
|
5 |
+
"startedAt": "2021-07-15T13:29:56.221807",
|
6 |
+
"docker": null,
|
7 |
+
"cpu_count": 96,
|
8 |
+
"cuda": null,
|
9 |
+
"args": [
|
10 |
+
"--push_to_hub",
|
11 |
+
"--output_dir=./",
|
12 |
+
"--model_type=big_bird",
|
13 |
+
"--config_name=./",
|
14 |
+
"--tokenizer_name=./",
|
15 |
+
"--max_seq_length=4096",
|
16 |
+
"--weight_decay=0.0095",
|
17 |
+
"--warmup_steps=10000",
|
18 |
+
"--overwrite_output_dir",
|
19 |
+
"--adam_beta1=0.9",
|
20 |
+
"--adam_beta2=0.98",
|
21 |
+
"--logging_steps=50",
|
22 |
+
"--eval_steps=2000",
|
23 |
+
"--num_train_epochs=2",
|
24 |
+
"--preprocessing_num_workers=96",
|
25 |
+
"--save_steps=10000",
|
26 |
+
"--learning_rate=3e-5",
|
27 |
+
"--per_device_train_batch_size=1",
|
28 |
+
"--per_device_eval_batch_size=1",
|
29 |
+
"--save_total_limit=5",
|
30 |
+
"--max_eval_samples=5000",
|
31 |
+
"--resume_from_checkpoint=./"
|
32 |
+
],
|
33 |
+
"state": "running",
|
34 |
+
"program": "./run_mlm_flax_no_accum.py",
|
35 |
+
"codePath": "run_mlm_flax_no_accum.py",
|
36 |
+
"git": {
|
37 |
+
"remote": "https://huggingface.co/flax-community/pino-roberta-base",
|
38 |
+
"commit": "b6814517994e86d862ddec99dda42b9dbea5b8b0"
|
39 |
+
},
|
40 |
+
"email": null,
|
41 |
+
"root": "/home/dat/pino-roberta-base",
|
42 |
+
"host": "t1v-n-f5c06ea1-w-0",
|
43 |
+
"username": "dat",
|
44 |
+
"executable": "/home/dat/pino/bin/python"
|
45 |
+
}
|
wandb/run-20210715_132956-37l0zsn1/files/wandb-summary.json
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
{}
|
wandb/run-20210715_132956-37l0zsn1/logs/debug-internal.log
ADDED
@@ -0,0 +1,391 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
2021-07-15 13:29:56,917 INFO MainThread:690242 [internal.py:wandb_internal():88] W&B internal server running at pid: 690242, started at: 2021-07-15 13:29:56.916849
|
2 |
+
2021-07-15 13:29:56,919 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: check_version
|
3 |
+
2021-07-15 13:29:56,919 INFO WriterThread:690242 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/run-37l0zsn1.wandb
|
4 |
+
2021-07-15 13:29:56,920 DEBUG SenderThread:690242 [sender.py:send():179] send: header
|
5 |
+
2021-07-15 13:29:56,920 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: check_version
|
6 |
+
2021-07-15 13:29:56,960 DEBUG SenderThread:690242 [sender.py:send():179] send: run
|
7 |
+
2021-07-15 13:29:57,133 INFO SenderThread:690242 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files
|
8 |
+
2021-07-15 13:29:57,133 INFO SenderThread:690242 [sender.py:_start_run_threads():716] run started: 37l0zsn1 with start time 1626355796
|
9 |
+
2021-07-15 13:29:57,133 DEBUG SenderThread:690242 [sender.py:send():179] send: summary
|
10 |
+
2021-07-15 13:29:57,133 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: run_start
|
11 |
+
2021-07-15 13:29:57,134 INFO SenderThread:690242 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
|
12 |
+
2021-07-15 13:29:58,138 INFO Thread-8 :690242 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/wandb-summary.json
|
13 |
+
2021-07-15 13:29:58,227 DEBUG HandlerThread:690242 [meta.py:__init__():39] meta init
|
14 |
+
2021-07-15 13:29:58,227 DEBUG HandlerThread:690242 [meta.py:__init__():53] meta init done
|
15 |
+
2021-07-15 13:29:58,227 DEBUG HandlerThread:690242 [meta.py:probe():210] probe
|
16 |
+
2021-07-15 13:29:58,228 DEBUG HandlerThread:690242 [meta.py:_setup_git():200] setup git
|
17 |
+
2021-07-15 13:29:58,258 DEBUG HandlerThread:690242 [meta.py:_setup_git():207] setup git done
|
18 |
+
2021-07-15 13:29:58,258 DEBUG HandlerThread:690242 [meta.py:_save_pip():57] save pip
|
19 |
+
2021-07-15 13:29:58,259 DEBUG HandlerThread:690242 [meta.py:_save_pip():71] save pip done
|
20 |
+
2021-07-15 13:29:58,259 DEBUG HandlerThread:690242 [meta.py:probe():252] probe done
|
21 |
+
2021-07-15 13:29:58,262 DEBUG SenderThread:690242 [sender.py:send():179] send: files
|
22 |
+
2021-07-15 13:29:58,262 INFO SenderThread:690242 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now
|
23 |
+
2021-07-15 13:29:58,270 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
24 |
+
2021-07-15 13:29:58,270 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
25 |
+
2021-07-15 13:29:58,400 DEBUG SenderThread:690242 [sender.py:send():179] send: config
|
26 |
+
2021-07-15 13:29:58,401 DEBUG SenderThread:690242 [sender.py:send():179] send: config
|
27 |
+
2021-07-15 13:29:58,401 DEBUG SenderThread:690242 [sender.py:send():179] send: config
|
28 |
+
2021-07-15 13:29:58,737 INFO Thread-11 :690242 [upload_job.py:push():137] Uploaded file /tmp/tmps2g86y4ywandb/37km806o-wandb-metadata.json
|
29 |
+
2021-07-15 13:29:59,137 INFO Thread-8 :690242 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
30 |
+
2021-07-15 13:29:59,138 INFO Thread-8 :690242 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/wandb-metadata.json
|
31 |
+
2021-07-15 13:29:59,138 INFO Thread-8 :690242 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/requirements.txt
|
32 |
+
2021-07-15 13:30:13,740 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
33 |
+
2021-07-15 13:30:13,740 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
34 |
+
2021-07-15 13:30:15,144 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
35 |
+
2021-07-15 13:30:17,144 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
36 |
+
2021-07-15 13:30:26,310 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
37 |
+
2021-07-15 13:30:28,149 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/config.yaml
|
38 |
+
2021-07-15 13:30:28,945 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
39 |
+
2021-07-15 13:30:28,945 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
40 |
+
2021-07-15 13:30:44,077 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
41 |
+
2021-07-15 13:30:44,077 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
42 |
+
2021-07-15 13:30:56,384 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
43 |
+
2021-07-15 13:30:59,224 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
44 |
+
2021-07-15 13:30:59,225 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
45 |
+
2021-07-15 13:31:14,355 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
46 |
+
2021-07-15 13:31:14,355 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
47 |
+
2021-07-15 13:31:26,457 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
48 |
+
2021-07-15 13:31:29,490 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
49 |
+
2021-07-15 13:31:29,490 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
50 |
+
2021-07-15 13:31:44,622 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
51 |
+
2021-07-15 13:31:44,623 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
52 |
+
2021-07-15 13:31:56,528 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
53 |
+
2021-07-15 13:31:59,753 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
54 |
+
2021-07-15 13:31:59,754 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
55 |
+
2021-07-15 13:32:14,887 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
56 |
+
2021-07-15 13:32:14,887 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
57 |
+
2021-07-15 13:32:26,603 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
58 |
+
2021-07-15 13:32:30,023 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
59 |
+
2021-07-15 13:32:30,024 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
60 |
+
2021-07-15 13:32:45,158 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
61 |
+
2021-07-15 13:32:45,158 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
62 |
+
2021-07-15 13:32:56,677 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
63 |
+
2021-07-15 13:33:00,291 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
64 |
+
2021-07-15 13:33:00,291 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
65 |
+
2021-07-15 13:33:15,424 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
66 |
+
2021-07-15 13:33:15,425 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
67 |
+
2021-07-15 13:33:26,741 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
68 |
+
2021-07-15 13:33:30,559 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
69 |
+
2021-07-15 13:33:30,559 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
70 |
+
2021-07-15 13:33:45,691 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
71 |
+
2021-07-15 13:33:45,692 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
72 |
+
2021-07-15 13:33:56,798 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
73 |
+
2021-07-15 13:34:00,825 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
74 |
+
2021-07-15 13:34:00,825 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
75 |
+
2021-07-15 13:34:15,959 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
76 |
+
2021-07-15 13:34:15,959 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
77 |
+
2021-07-15 13:34:26,856 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
78 |
+
2021-07-15 13:34:31,092 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
79 |
+
2021-07-15 13:34:31,093 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
80 |
+
2021-07-15 13:34:46,236 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
81 |
+
2021-07-15 13:34:46,236 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
82 |
+
2021-07-15 13:34:56,922 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
83 |
+
2021-07-15 13:35:01,377 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
84 |
+
2021-07-15 13:35:01,377 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
85 |
+
2021-07-15 13:35:16,510 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
86 |
+
2021-07-15 13:35:16,511 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
87 |
+
2021-07-15 13:35:26,998 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
88 |
+
2021-07-15 13:35:31,642 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
89 |
+
2021-07-15 13:35:31,642 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
90 |
+
2021-07-15 13:35:45,257 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
91 |
+
2021-07-15 13:35:46,786 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
92 |
+
2021-07-15 13:35:46,787 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
93 |
+
2021-07-15 13:35:57,072 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
94 |
+
2021-07-15 13:36:01,936 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
95 |
+
2021-07-15 13:36:01,936 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
96 |
+
2021-07-15 13:36:17,084 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
97 |
+
2021-07-15 13:36:17,084 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
98 |
+
2021-07-15 13:36:27,147 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
99 |
+
2021-07-15 13:36:32,219 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
100 |
+
2021-07-15 13:36:32,219 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
101 |
+
2021-07-15 13:36:47,351 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
102 |
+
2021-07-15 13:36:47,351 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
103 |
+
2021-07-15 13:36:57,222 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
104 |
+
2021-07-15 13:37:02,481 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
105 |
+
2021-07-15 13:37:02,482 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
106 |
+
2021-07-15 13:37:17,616 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
107 |
+
2021-07-15 13:37:17,617 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
108 |
+
2021-07-15 13:37:27,306 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
109 |
+
2021-07-15 13:37:32,749 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
110 |
+
2021-07-15 13:37:32,749 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
111 |
+
2021-07-15 13:37:45,303 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
112 |
+
2021-07-15 13:37:47,971 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
113 |
+
2021-07-15 13:37:47,971 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
114 |
+
2021-07-15 13:37:57,376 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
115 |
+
2021-07-15 13:38:03,134 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
116 |
+
2021-07-15 13:38:03,135 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
117 |
+
2021-07-15 13:38:18,276 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
118 |
+
2021-07-15 13:38:18,277 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
119 |
+
2021-07-15 13:38:27,456 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
120 |
+
2021-07-15 13:38:33,418 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
121 |
+
2021-07-15 13:38:33,419 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
122 |
+
2021-07-15 13:38:48,552 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
123 |
+
2021-07-15 13:38:48,553 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
124 |
+
2021-07-15 13:38:57,533 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
125 |
+
2021-07-15 13:39:03,683 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
126 |
+
2021-07-15 13:39:03,683 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
127 |
+
2021-07-15 13:39:18,813 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
128 |
+
2021-07-15 13:39:18,814 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
129 |
+
2021-07-15 13:39:27,608 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
130 |
+
2021-07-15 13:39:34,015 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
131 |
+
2021-07-15 13:39:34,015 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
132 |
+
2021-07-15 13:39:49,151 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
133 |
+
2021-07-15 13:39:49,152 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
134 |
+
2021-07-15 13:39:57,698 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
135 |
+
2021-07-15 13:40:01,356 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
136 |
+
2021-07-15 13:40:03,357 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
137 |
+
2021-07-15 13:40:04,295 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
138 |
+
2021-07-15 13:40:04,296 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
139 |
+
2021-07-15 13:40:05,357 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
140 |
+
2021-07-15 13:40:07,358 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
141 |
+
2021-07-15 13:40:09,359 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
142 |
+
2021-07-15 13:40:12,361 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
143 |
+
2021-07-15 13:40:14,362 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
144 |
+
2021-07-15 13:40:16,362 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
145 |
+
2021-07-15 13:40:18,363 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
146 |
+
2021-07-15 13:40:19,564 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
147 |
+
2021-07-15 13:40:19,564 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
148 |
+
2021-07-15 13:40:20,364 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
149 |
+
2021-07-15 13:40:22,365 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
150 |
+
2021-07-15 13:40:24,366 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
151 |
+
2021-07-15 13:40:26,367 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
152 |
+
2021-07-15 13:40:27,772 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
153 |
+
2021-07-15 13:40:28,367 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
154 |
+
2021-07-15 13:40:30,368 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
155 |
+
2021-07-15 13:40:32,369 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
156 |
+
2021-07-15 13:40:34,371 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
157 |
+
2021-07-15 13:40:34,696 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
158 |
+
2021-07-15 13:40:34,697 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
159 |
+
2021-07-15 13:40:36,372 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
160 |
+
2021-07-15 13:40:38,373 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
161 |
+
2021-07-15 13:40:40,374 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
162 |
+
2021-07-15 13:40:42,374 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
163 |
+
2021-07-15 13:40:44,375 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
164 |
+
2021-07-15 13:40:46,376 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
165 |
+
2021-07-15 13:40:48,377 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
166 |
+
2021-07-15 13:40:49,831 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
167 |
+
2021-07-15 13:40:49,831 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
168 |
+
2021-07-15 13:40:50,378 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
169 |
+
2021-07-15 13:40:52,379 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
170 |
+
2021-07-15 13:40:54,380 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
171 |
+
2021-07-15 13:40:56,381 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
172 |
+
2021-07-15 13:40:57,856 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
173 |
+
2021-07-15 13:40:58,382 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
174 |
+
2021-07-15 13:41:00,383 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
175 |
+
2021-07-15 13:41:02,384 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
176 |
+
2021-07-15 13:41:04,385 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
177 |
+
2021-07-15 13:41:04,961 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
178 |
+
2021-07-15 13:41:04,961 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
179 |
+
2021-07-15 13:41:06,386 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
180 |
+
2021-07-15 13:41:08,387 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
181 |
+
2021-07-15 13:41:10,388 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
182 |
+
2021-07-15 13:41:12,389 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
183 |
+
2021-07-15 13:41:14,390 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
184 |
+
2021-07-15 13:41:16,391 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
185 |
+
2021-07-15 13:41:18,392 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
186 |
+
2021-07-15 13:41:20,092 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
187 |
+
2021-07-15 13:41:20,092 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
188 |
+
2021-07-15 13:41:20,393 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
189 |
+
2021-07-15 13:41:22,394 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
190 |
+
2021-07-15 13:41:24,395 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
191 |
+
2021-07-15 13:41:26,396 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
192 |
+
2021-07-15 13:41:27,927 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
193 |
+
2021-07-15 13:41:28,396 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
194 |
+
2021-07-15 13:41:30,397 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
195 |
+
2021-07-15 13:41:32,398 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
196 |
+
2021-07-15 13:41:34,399 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
197 |
+
2021-07-15 13:41:35,222 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
198 |
+
2021-07-15 13:41:35,222 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
199 |
+
2021-07-15 13:41:36,400 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
200 |
+
2021-07-15 13:41:38,401 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
201 |
+
2021-07-15 13:41:40,402 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
202 |
+
2021-07-15 13:41:42,403 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
203 |
+
2021-07-15 13:41:44,404 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
204 |
+
2021-07-15 13:41:46,405 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
205 |
+
2021-07-15 13:41:48,406 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
206 |
+
2021-07-15 13:41:50,352 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
207 |
+
2021-07-15 13:41:50,352 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
208 |
+
2021-07-15 13:41:50,407 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
209 |
+
2021-07-15 13:41:52,408 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
210 |
+
2021-07-15 13:41:54,409 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
211 |
+
2021-07-15 13:41:56,409 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
212 |
+
2021-07-15 13:41:57,995 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
213 |
+
2021-07-15 13:41:58,410 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
214 |
+
2021-07-15 13:42:00,411 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
215 |
+
2021-07-15 13:42:02,412 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
216 |
+
2021-07-15 13:42:04,413 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
217 |
+
2021-07-15 13:42:05,483 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
218 |
+
2021-07-15 13:42:05,484 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
219 |
+
2021-07-15 13:42:06,413 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
220 |
+
2021-07-15 13:42:08,414 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
221 |
+
2021-07-15 13:42:10,415 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
222 |
+
2021-07-15 13:42:12,416 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
223 |
+
2021-07-15 13:42:14,417 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
224 |
+
2021-07-15 13:42:16,418 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
225 |
+
2021-07-15 13:42:18,418 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
226 |
+
2021-07-15 13:42:20,419 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
227 |
+
2021-07-15 13:42:20,615 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
228 |
+
2021-07-15 13:42:20,616 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
229 |
+
2021-07-15 13:42:22,420 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
230 |
+
2021-07-15 13:42:24,421 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
231 |
+
2021-07-15 13:42:26,422 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
232 |
+
2021-07-15 13:42:28,066 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
233 |
+
2021-07-15 13:42:28,422 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
234 |
+
2021-07-15 13:42:30,423 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
235 |
+
2021-07-15 13:42:32,424 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
236 |
+
2021-07-15 13:42:34,425 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
237 |
+
2021-07-15 13:42:35,983 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
238 |
+
2021-07-15 13:42:35,983 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
239 |
+
2021-07-15 13:42:36,426 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
240 |
+
2021-07-15 13:42:38,427 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
241 |
+
2021-07-15 13:42:40,427 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
242 |
+
2021-07-15 13:42:42,428 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
243 |
+
2021-07-15 13:42:44,429 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
244 |
+
2021-07-15 13:42:46,430 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
245 |
+
2021-07-15 13:42:48,431 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
246 |
+
2021-07-15 13:42:50,432 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
247 |
+
2021-07-15 13:42:51,117 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
248 |
+
2021-07-15 13:42:51,117 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
249 |
+
2021-07-15 13:42:52,433 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
250 |
+
2021-07-15 13:42:54,434 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
251 |
+
2021-07-15 13:42:56,435 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
252 |
+
2021-07-15 13:42:58,133 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
253 |
+
2021-07-15 13:42:58,435 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
254 |
+
2021-07-15 13:43:00,436 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
255 |
+
2021-07-15 13:43:02,437 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
256 |
+
2021-07-15 13:43:04,438 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
257 |
+
2021-07-15 13:43:06,248 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
258 |
+
2021-07-15 13:43:06,249 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
259 |
+
2021-07-15 13:43:06,439 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
260 |
+
2021-07-15 13:43:08,440 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
261 |
+
2021-07-15 13:43:10,441 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
262 |
+
2021-07-15 13:43:12,442 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
263 |
+
2021-07-15 13:43:14,443 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
264 |
+
2021-07-15 13:43:16,444 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
265 |
+
2021-07-15 13:43:18,445 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
266 |
+
2021-07-15 13:43:20,446 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
267 |
+
2021-07-15 13:43:21,382 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
268 |
+
2021-07-15 13:43:21,383 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
269 |
+
2021-07-15 13:43:22,446 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
270 |
+
2021-07-15 13:43:24,447 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
271 |
+
2021-07-15 13:43:26,448 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
272 |
+
2021-07-15 13:43:28,203 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
273 |
+
2021-07-15 13:43:28,449 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
274 |
+
2021-07-15 13:43:30,450 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
275 |
+
2021-07-15 13:43:32,451 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
276 |
+
2021-07-15 13:43:34,452 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
277 |
+
2021-07-15 13:43:36,453 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
278 |
+
2021-07-15 13:43:36,515 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
279 |
+
2021-07-15 13:43:36,516 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
280 |
+
2021-07-15 13:43:38,453 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
281 |
+
2021-07-15 13:43:40,454 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
282 |
+
2021-07-15 13:43:42,455 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
283 |
+
2021-07-15 13:43:44,456 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
284 |
+
2021-07-15 13:43:46,457 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
285 |
+
2021-07-15 13:43:48,458 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
286 |
+
2021-07-15 13:43:50,459 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
287 |
+
2021-07-15 13:43:51,649 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
288 |
+
2021-07-15 13:43:51,649 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
289 |
+
2021-07-15 13:43:52,460 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
290 |
+
2021-07-15 13:43:54,461 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
291 |
+
2021-07-15 13:43:56,462 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
292 |
+
2021-07-15 13:43:58,275 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
293 |
+
2021-07-15 13:43:58,463 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
294 |
+
2021-07-15 13:44:00,464 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
295 |
+
2021-07-15 13:44:02,465 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
296 |
+
2021-07-15 13:44:04,466 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
297 |
+
2021-07-15 13:44:06,467 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
298 |
+
2021-07-15 13:44:06,780 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
299 |
+
2021-07-15 13:44:06,781 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
300 |
+
2021-07-15 13:44:08,468 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
301 |
+
2021-07-15 13:44:10,469 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
302 |
+
2021-07-15 13:44:12,469 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
303 |
+
2021-07-15 13:44:14,470 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
304 |
+
2021-07-15 13:44:16,471 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
305 |
+
2021-07-15 13:44:18,472 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
306 |
+
2021-07-15 13:44:20,473 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
307 |
+
2021-07-15 13:44:21,910 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
308 |
+
2021-07-15 13:44:21,911 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
309 |
+
2021-07-15 13:44:22,474 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
310 |
+
2021-07-15 13:44:24,474 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
311 |
+
2021-07-15 13:44:26,475 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
312 |
+
2021-07-15 13:44:28,347 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
313 |
+
2021-07-15 13:44:28,476 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
314 |
+
2021-07-15 13:44:30,477 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
315 |
+
2021-07-15 13:44:32,477 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
316 |
+
2021-07-15 13:44:34,478 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
317 |
+
2021-07-15 13:44:36,479 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
318 |
+
2021-07-15 13:44:37,042 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
319 |
+
2021-07-15 13:44:37,042 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
320 |
+
2021-07-15 13:44:38,480 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
321 |
+
2021-07-15 13:44:40,481 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
322 |
+
2021-07-15 13:44:42,482 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
323 |
+
2021-07-15 13:44:44,482 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
324 |
+
2021-07-15 13:44:46,483 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
325 |
+
2021-07-15 13:44:48,484 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
326 |
+
2021-07-15 13:44:50,485 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
327 |
+
2021-07-15 13:44:52,173 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
328 |
+
2021-07-15 13:44:52,174 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
329 |
+
2021-07-15 13:44:52,486 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
330 |
+
2021-07-15 13:44:54,486 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
331 |
+
2021-07-15 13:44:56,487 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
332 |
+
2021-07-15 13:44:58,423 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
333 |
+
2021-07-15 13:44:58,488 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
334 |
+
2021-07-15 13:45:00,489 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
335 |
+
2021-07-15 13:45:02,489 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
336 |
+
2021-07-15 13:45:04,490 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
337 |
+
2021-07-15 13:45:06,491 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
338 |
+
2021-07-15 13:45:07,305 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
339 |
+
2021-07-15 13:45:07,305 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
340 |
+
2021-07-15 13:45:08,492 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
341 |
+
2021-07-15 13:45:10,493 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
342 |
+
2021-07-15 13:45:12,493 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
343 |
+
2021-07-15 13:45:14,494 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
344 |
+
2021-07-15 13:45:16,495 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
345 |
+
2021-07-15 13:45:18,496 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
346 |
+
2021-07-15 13:45:20,496 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
347 |
+
2021-07-15 13:45:22,436 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
348 |
+
2021-07-15 13:45:22,436 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
349 |
+
2021-07-15 13:45:22,497 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
350 |
+
2021-07-15 13:45:24,498 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
351 |
+
2021-07-15 13:45:27,500 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
352 |
+
2021-07-15 13:45:28,499 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
353 |
+
2021-07-15 13:45:29,502 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
354 |
+
2021-07-15 13:45:31,503 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
355 |
+
2021-07-15 13:45:33,504 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
356 |
+
2021-07-15 13:45:35,504 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
357 |
+
2021-07-15 13:45:37,505 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
358 |
+
2021-07-15 13:45:37,572 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
359 |
+
2021-07-15 13:45:37,572 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
360 |
+
2021-07-15 13:45:39,506 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
361 |
+
2021-07-15 13:45:41,507 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
362 |
+
2021-07-15 13:45:43,508 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
363 |
+
2021-07-15 13:45:45,509 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
364 |
+
2021-07-15 13:45:47,510 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
365 |
+
2021-07-15 13:45:49,511 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
366 |
+
2021-07-15 13:45:51,512 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
367 |
+
2021-07-15 13:45:52,704 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
368 |
+
2021-07-15 13:45:52,705 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
369 |
+
2021-07-15 13:45:53,512 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
370 |
+
2021-07-15 13:45:55,513 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
371 |
+
2021-07-15 13:45:57,514 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
372 |
+
2021-07-15 13:45:58,571 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
|
373 |
+
2021-07-15 13:45:59,515 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
374 |
+
2021-07-15 13:46:01,516 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
375 |
+
2021-07-15 13:46:03,516 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
376 |
+
2021-07-15 13:46:05,517 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
377 |
+
2021-07-15 13:46:07,518 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
378 |
+
2021-07-15 13:46:07,841 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
|
379 |
+
2021-07-15 13:46:07,841 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
|
380 |
+
2021-07-15 13:46:09,519 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
381 |
+
2021-07-15 13:46:11,520 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
382 |
+
2021-07-15 13:46:13,520 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
383 |
+
2021-07-15 13:46:15,521 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
|
384 |
+
2021-07-15 13:46:15,809 WARNING MainThread:690242 [internal.py:wandb_internal():147] Internal process interrupt: 1
|
385 |
+
2021-07-15 13:46:15,991 WARNING MainThread:690242 [internal.py:wandb_internal():147] Internal process interrupt: 2
|
386 |
+
2021-07-15 13:46:15,991 ERROR MainThread:690242 [internal.py:wandb_internal():150] Internal process interrupted.
|
387 |
+
2021-07-15 13:46:16,587 INFO SenderThread:690242 [sender.py:finish():945] shutting down sender
|
388 |
+
2021-07-15 13:46:16,587 INFO SenderThread:690242 [dir_watcher.py:finish():282] shutting down directory watcher
|
389 |
+
2021-07-15 13:46:16,588 INFO WriterThread:690242 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/run-37l0zsn1.wandb
|
390 |
+
2021-07-15 13:46:16,589 INFO HandlerThread:690242 [handler.py:finish():638] shutting down handler
|
391 |
+
2021-07-15 13:46:16,860 INFO MainThread:690242 [internal.py:handle_exit():78] Internal process exited
|
wandb/run-20210715_132956-37l0zsn1/logs/debug.log
ADDED
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
2021-07-15 13:29:56,223 INFO MainThread:688983 [wandb_setup.py:_flush():69] setting env: {}
|
2 |
+
2021-07-15 13:29:56,223 INFO MainThread:688983 [wandb_setup.py:_flush():69] setting login settings: {}
|
3 |
+
2021-07-15 13:29:56,223 INFO MainThread:688983 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/logs/debug.log
|
4 |
+
2021-07-15 13:29:56,223 INFO MainThread:688983 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/logs/debug-internal.log
|
5 |
+
2021-07-15 13:29:56,223 INFO MainThread:688983 [wandb_init.py:init():370] calling init triggers
|
6 |
+
2021-07-15 13:29:56,223 INFO MainThread:688983 [wandb_init.py:init():375] wandb.init called with sweep_config: {}
|
7 |
+
config: {}
|
8 |
+
2021-07-15 13:29:56,224 INFO MainThread:688983 [wandb_init.py:init():419] starting backend
|
9 |
+
2021-07-15 13:29:56,224 INFO MainThread:688983 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
|
10 |
+
2021-07-15 13:29:56,272 INFO MainThread:688983 [backend.py:ensure_launched():135] starting backend process...
|
11 |
+
2021-07-15 13:29:56,319 INFO MainThread:688983 [backend.py:ensure_launched():139] started backend process with pid: 690242
|
12 |
+
2021-07-15 13:29:56,321 INFO MainThread:688983 [wandb_init.py:init():424] backend started and connected
|
13 |
+
2021-07-15 13:29:56,324 INFO MainThread:688983 [wandb_init.py:init():472] updated telemetry
|
14 |
+
2021-07-15 13:29:56,325 INFO MainThread:688983 [wandb_init.py:init():491] communicating current version
|
15 |
+
2021-07-15 13:29:56,958 INFO MainThread:688983 [wandb_init.py:init():496] got version response
|
16 |
+
2021-07-15 13:29:56,959 INFO MainThread:688983 [wandb_init.py:init():504] communicating run to backend with 30 second timeout
|
17 |
+
2021-07-15 13:29:57,133 INFO MainThread:688983 [wandb_init.py:init():529] starting run threads in backend
|
18 |
+
2021-07-15 13:29:58,265 INFO MainThread:688983 [wandb_run.py:_console_start():1623] atexit reg
|
19 |
+
2021-07-15 13:29:58,266 INFO MainThread:688983 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT
|
20 |
+
2021-07-15 13:29:58,266 INFO MainThread:688983 [wandb_run.py:_redirect():1502] Redirecting console.
|
21 |
+
2021-07-15 13:29:58,268 INFO MainThread:688983 [wandb_run.py:_redirect():1558] Redirects installed.
|
22 |
+
2021-07-15 13:29:58,268 INFO MainThread:688983 [wandb_init.py:init():554] run started, returning control to user process
|
23 |
+
2021-07-15 13:29:58,274 INFO MainThread:688983 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 3e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 2.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul15_13-29-47_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 50, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 10000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 2000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': './', 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''}
|
24 |
+
2021-07-15 13:29:58,276 INFO MainThread:688983 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'}
|
25 |
+
2021-07-15 13:29:58,278 INFO MainThread:688983 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 5000}
|
26 |
+
2021-07-15 13:46:15,841 INFO MainThread:688983 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 255
|
27 |
+
2021-07-15 13:46:15,841 INFO MainThread:688983 [wandb_run.py:_restore():1565] restore
|
wandb/run-20210715_132956-37l0zsn1/run-37l0zsn1.wandb
ADDED
Binary file (64.3 kB). View file
|
|
wandb/run-20210715_135329-1c6qs9ww/files/config.yaml
ADDED
@@ -0,0 +1,301 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
wandb_version: 1
|
2 |
+
|
3 |
+
_wandb:
|
4 |
+
desc: null
|
5 |
+
value:
|
6 |
+
cli_version: 0.10.33
|
7 |
+
framework: huggingface
|
8 |
+
huggingface_version: 4.9.0.dev0
|
9 |
+
is_jupyter_run: false
|
10 |
+
is_kaggle_kernel: false
|
11 |
+
python_version: 3.8.10
|
12 |
+
t:
|
13 |
+
1:
|
14 |
+
- 3
|
15 |
+
- 11
|
16 |
+
4: 3.8.10
|
17 |
+
5: 0.10.33
|
18 |
+
6: 4.9.0.dev0
|
19 |
+
8:
|
20 |
+
- 5
|
21 |
+
adafactor:
|
22 |
+
desc: null
|
23 |
+
value: false
|
24 |
+
adam_beta1:
|
25 |
+
desc: null
|
26 |
+
value: 0.9
|
27 |
+
adam_beta2:
|
28 |
+
desc: null
|
29 |
+
value: 0.98
|
30 |
+
adam_epsilon:
|
31 |
+
desc: null
|
32 |
+
value: 1.0e-08
|
33 |
+
cache_dir:
|
34 |
+
desc: null
|
35 |
+
value: null
|
36 |
+
config_name:
|
37 |
+
desc: null
|
38 |
+
value: ./
|
39 |
+
dataloader_drop_last:
|
40 |
+
desc: null
|
41 |
+
value: false
|
42 |
+
dataloader_num_workers:
|
43 |
+
desc: null
|
44 |
+
value: 0
|
45 |
+
dataloader_pin_memory:
|
46 |
+
desc: null
|
47 |
+
value: true
|
48 |
+
dataset_config_name:
|
49 |
+
desc: null
|
50 |
+
value: null
|
51 |
+
dataset_name:
|
52 |
+
desc: null
|
53 |
+
value: null
|
54 |
+
ddp_find_unused_parameters:
|
55 |
+
desc: null
|
56 |
+
value: null
|
57 |
+
debug:
|
58 |
+
desc: null
|
59 |
+
value: []
|
60 |
+
deepspeed:
|
61 |
+
desc: null
|
62 |
+
value: null
|
63 |
+
disable_tqdm:
|
64 |
+
desc: null
|
65 |
+
value: false
|
66 |
+
do_eval:
|
67 |
+
desc: null
|
68 |
+
value: false
|
69 |
+
do_predict:
|
70 |
+
desc: null
|
71 |
+
value: false
|
72 |
+
do_train:
|
73 |
+
desc: null
|
74 |
+
value: false
|
75 |
+
dtype:
|
76 |
+
desc: null
|
77 |
+
value: float32
|
78 |
+
eval_accumulation_steps:
|
79 |
+
desc: null
|
80 |
+
value: null
|
81 |
+
eval_steps:
|
82 |
+
desc: null
|
83 |
+
value: 2000
|
84 |
+
evaluation_strategy:
|
85 |
+
desc: null
|
86 |
+
value: IntervalStrategy.NO
|
87 |
+
fp16:
|
88 |
+
desc: null
|
89 |
+
value: false
|
90 |
+
fp16_backend:
|
91 |
+
desc: null
|
92 |
+
value: auto
|
93 |
+
fp16_full_eval:
|
94 |
+
desc: null
|
95 |
+
value: false
|
96 |
+
fp16_opt_level:
|
97 |
+
desc: null
|
98 |
+
value: O1
|
99 |
+
gradient_accumulation_steps:
|
100 |
+
desc: null
|
101 |
+
value: 1
|
102 |
+
greater_is_better:
|
103 |
+
desc: null
|
104 |
+
value: null
|
105 |
+
group_by_length:
|
106 |
+
desc: null
|
107 |
+
value: false
|
108 |
+
ignore_data_skip:
|
109 |
+
desc: null
|
110 |
+
value: false
|
111 |
+
label_names:
|
112 |
+
desc: null
|
113 |
+
value: null
|
114 |
+
label_smoothing_factor:
|
115 |
+
desc: null
|
116 |
+
value: 0.0
|
117 |
+
learning_rate:
|
118 |
+
desc: null
|
119 |
+
value: 3.0e-05
|
120 |
+
length_column_name:
|
121 |
+
desc: null
|
122 |
+
value: length
|
123 |
+
line_by_line:
|
124 |
+
desc: null
|
125 |
+
value: false
|
126 |
+
load_best_model_at_end:
|
127 |
+
desc: null
|
128 |
+
value: false
|
129 |
+
local_rank:
|
130 |
+
desc: null
|
131 |
+
value: -1
|
132 |
+
log_level:
|
133 |
+
desc: null
|
134 |
+
value: -1
|
135 |
+
log_level_replica:
|
136 |
+
desc: null
|
137 |
+
value: -1
|
138 |
+
log_on_each_node:
|
139 |
+
desc: null
|
140 |
+
value: true
|
141 |
+
logging_dir:
|
142 |
+
desc: null
|
143 |
+
value: ./runs/Jul15_13-53-21_t1v-n-f5c06ea1-w-0
|
144 |
+
logging_first_step:
|
145 |
+
desc: null
|
146 |
+
value: false
|
147 |
+
logging_steps:
|
148 |
+
desc: null
|
149 |
+
value: 50
|
150 |
+
logging_strategy:
|
151 |
+
desc: null
|
152 |
+
value: IntervalStrategy.STEPS
|
153 |
+
lr_scheduler_type:
|
154 |
+
desc: null
|
155 |
+
value: SchedulerType.LINEAR
|
156 |
+
max_eval_samples:
|
157 |
+
desc: null
|
158 |
+
value: 5000
|
159 |
+
max_grad_norm:
|
160 |
+
desc: null
|
161 |
+
value: 1.0
|
162 |
+
max_seq_length:
|
163 |
+
desc: null
|
164 |
+
value: 4096
|
165 |
+
max_steps:
|
166 |
+
desc: null
|
167 |
+
value: -1
|
168 |
+
metric_for_best_model:
|
169 |
+
desc: null
|
170 |
+
value: null
|
171 |
+
mlm_probability:
|
172 |
+
desc: null
|
173 |
+
value: 0.15
|
174 |
+
model_name_or_path:
|
175 |
+
desc: null
|
176 |
+
value: null
|
177 |
+
model_type:
|
178 |
+
desc: null
|
179 |
+
value: big_bird
|
180 |
+
mp_parameters:
|
181 |
+
desc: null
|
182 |
+
value: ''
|
183 |
+
no_cuda:
|
184 |
+
desc: null
|
185 |
+
value: false
|
186 |
+
num_train_epochs:
|
187 |
+
desc: null
|
188 |
+
value: 2.0
|
189 |
+
output_dir:
|
190 |
+
desc: null
|
191 |
+
value: ./
|
192 |
+
overwrite_cache:
|
193 |
+
desc: null
|
194 |
+
value: false
|
195 |
+
overwrite_output_dir:
|
196 |
+
desc: null
|
197 |
+
value: true
|
198 |
+
pad_to_max_length:
|
199 |
+
desc: null
|
200 |
+
value: false
|
201 |
+
past_index:
|
202 |
+
desc: null
|
203 |
+
value: -1
|
204 |
+
per_device_eval_batch_size:
|
205 |
+
desc: null
|
206 |
+
value: 1
|
207 |
+
per_device_train_batch_size:
|
208 |
+
desc: null
|
209 |
+
value: 1
|
210 |
+
per_gpu_eval_batch_size:
|
211 |
+
desc: null
|
212 |
+
value: null
|
213 |
+
per_gpu_train_batch_size:
|
214 |
+
desc: null
|
215 |
+
value: null
|
216 |
+
prediction_loss_only:
|
217 |
+
desc: null
|
218 |
+
value: false
|
219 |
+
preprocessing_num_workers:
|
220 |
+
desc: null
|
221 |
+
value: 96
|
222 |
+
push_to_hub:
|
223 |
+
desc: null
|
224 |
+
value: true
|
225 |
+
push_to_hub_model_id:
|
226 |
+
desc: null
|
227 |
+
value: ''
|
228 |
+
push_to_hub_organization:
|
229 |
+
desc: null
|
230 |
+
value: null
|
231 |
+
push_to_hub_token:
|
232 |
+
desc: null
|
233 |
+
value: null
|
234 |
+
remove_unused_columns:
|
235 |
+
desc: null
|
236 |
+
value: true
|
237 |
+
report_to:
|
238 |
+
desc: null
|
239 |
+
value:
|
240 |
+
- tensorboard
|
241 |
+
- wandb
|
242 |
+
resume_from_checkpoint:
|
243 |
+
desc: null
|
244 |
+
value: ./
|
245 |
+
run_name:
|
246 |
+
desc: null
|
247 |
+
value: ./
|
248 |
+
save_on_each_node:
|
249 |
+
desc: null
|
250 |
+
value: false
|
251 |
+
save_steps:
|
252 |
+
desc: null
|
253 |
+
value: 10000
|
254 |
+
save_strategy:
|
255 |
+
desc: null
|
256 |
+
value: IntervalStrategy.STEPS
|
257 |
+
save_total_limit:
|
258 |
+
desc: null
|
259 |
+
value: 5
|
260 |
+
seed:
|
261 |
+
desc: null
|
262 |
+
value: 42
|
263 |
+
sharded_ddp:
|
264 |
+
desc: null
|
265 |
+
value: []
|
266 |
+
skip_memory_metrics:
|
267 |
+
desc: null
|
268 |
+
value: true
|
269 |
+
tokenizer_name:
|
270 |
+
desc: null
|
271 |
+
value: ./
|
272 |
+
tpu_metrics_debug:
|
273 |
+
desc: null
|
274 |
+
value: false
|
275 |
+
tpu_num_cores:
|
276 |
+
desc: null
|
277 |
+
value: null
|
278 |
+
train_ref_file:
|
279 |
+
desc: null
|
280 |
+
value: null
|
281 |
+
use_fast_tokenizer:
|
282 |
+
desc: null
|
283 |
+
value: true
|
284 |
+
use_legacy_prediction_loop:
|
285 |
+
desc: null
|
286 |
+
value: false
|
287 |
+
validation_ref_file:
|
288 |
+
desc: null
|
289 |
+
value: null
|
290 |
+
validation_split_percentage:
|
291 |
+
desc: null
|
292 |
+
value: 5
|
293 |
+
warmup_ratio:
|
294 |
+
desc: null
|
295 |
+
value: 0.0
|
296 |
+
warmup_steps:
|
297 |
+
desc: null
|
298 |
+
value: 10000
|
299 |
+
weight_decay:
|
300 |
+
desc: null
|
301 |
+
value: 0.0095
|
wandb/run-20210715_135329-1c6qs9ww/files/output.log
ADDED
@@ -0,0 +1,119 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
[13:53:43] - INFO - absl - Restoring checkpoint from ./checkpoint_90000
|
2 |
+
tcmalloc: large alloc 1530273792 bytes == 0x9ad7a000 @ 0x7f3feb16d680 0x7f3feb18e824 0x5b9a14 0x50b2ae 0x50cb1b 0x5a6f17 0x5f3010 0x56fd36 0x568d9a 0x5f5b33 0x56aadf 0x568d9a 0x68cdc7 0x67e161 0x67e1df 0x67e281 0x67e627 0x6b6e62 0x6b71ed 0x7f3feaf820b3 0x5f96de
|
3 |
+
/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:386: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code.
|
4 |
+
warnings.warn(
|
5 |
+
/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:373: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code.
|
6 |
+
warnings.warn(
|
7 |
+
Epoch ... (1/2): 0%| | 0/2 [00:00<?, ?it/s]
|
8 |
+
|
9 |
+
|
10 |
+
|
11 |
+
|
12 |
+
|
13 |
+
Training...: 26%|████████████████████████████████████▊ | 90051/352766 [04:22<27:51:51, 2.62it/s]
|
14 |
+
|
15 |
+
|
16 |
+
|
17 |
+
|
18 |
+
Training...: 26%|████████████████████████████████████▊ | 90101/352766 [04:42<27:17:08, 2.67it/s]
|
19 |
+
|
20 |
+
|
21 |
+
|
22 |
+
|
23 |
+
Training...: 26%|████████████████████████████████████▊ | 90151/352766 [05:02<27:16:16, 2.67it/s]
|
24 |
+
|
25 |
+
|
26 |
+
|
27 |
+
|
28 |
+
Training...: 26%|████████████████████████████████████▊ | 90201/352766 [05:22<27:14:35, 2.68it/s]
|
29 |
+
|
30 |
+
|
31 |
+
|
32 |
+
|
33 |
+
Training...: 26%|████████████████████████████████████▊ | 90251/352766 [05:42<27:01:30, 2.70it/s]
|
34 |
+
|
35 |
+
|
36 |
+
|
37 |
+
|
38 |
+
|
39 |
+
Training...: 26%|████████████████████████████████████▌ | 90302/352766 [06:15<292:20:48, 4.01s/it]
|
40 |
+
|
41 |
+
|
42 |
+
|
43 |
+
|
44 |
+
Training...: 26%|████████████████████████████████████▉ | 90351/352766 [06:22<26:58:26, 2.70it/s]
|
45 |
+
|
46 |
+
|
47 |
+
|
48 |
+
|
49 |
+
Training...: 26%|████████████████████████████████████▉ | 90401/352766 [06:42<27:14:02, 2.68it/s]
|
50 |
+
|
51 |
+
|
52 |
+
|
53 |
+
|
54 |
+
Training...: 26%|████████████████████████████████████▉ | 90451/352766 [07:02<27:03:56, 2.69it/s]
|
55 |
+
|
56 |
+
|
57 |
+
|
58 |
+
|
59 |
+
Training...: 26%|████████████████████████████████████▉ | 90501/352766 [07:22<27:12:39, 2.68it/s]
|
60 |
+
|
61 |
+
|
62 |
+
|
63 |
+
|
64 |
+
|
65 |
+
Training...: 26%|████████████████████████████████████▋ | 90552/352766 [07:55<291:55:56, 4.01s/it]
|
66 |
+
|
67 |
+
|
68 |
+
|
69 |
+
|
70 |
+
|
71 |
+
Training...: 26%|████████████████████████████████████▋ | 90604/352766 [08:16<163:26:36, 2.24s/it]
|
72 |
+
|
73 |
+
|
74 |
+
|
75 |
+
|
76 |
+
|
77 |
+
Training...: 26%|████████████████████████████████████▋ | 90654/352766 [08:36<163:24:41, 2.24s/it]
|
78 |
+
|
79 |
+
|
80 |
+
|
81 |
+
|
82 |
+
|
83 |
+
Training...: 26%|████████████████████████████████████▊ | 90706/352766 [08:56<101:36:18, 1.40s/it]
|
84 |
+
|
85 |
+
|
86 |
+
|
87 |
+
|
88 |
+
|
89 |
+
Training...: 26%|████████████████���███████████████████▊ | 90752/352766 [09:16<291:41:06, 4.01s/it]
|
90 |
+
|
91 |
+
|
92 |
+
|
93 |
+
|
94 |
+
|
95 |
+
Training...: 26%|████████████████████████████████████▊ | 90802/352766 [09:36<291:45:30, 4.01s/it]
|
96 |
+
|
97 |
+
|
98 |
+
|
99 |
+
|
100 |
+
|
101 |
+
Training...: 26%|████████████████████████████████████▊ | 90852/352766 [09:56<291:42:28, 4.01s/it]
|
102 |
+
|
103 |
+
|
104 |
+
|
105 |
+
|
106 |
+
|
107 |
+
Training...: 26%|████████████████████████████████████▊ | 90906/352766 [10:16<101:52:03, 1.40s/it]
|
108 |
+
|
109 |
+
|
110 |
+
|
111 |
+
|
112 |
+
|
113 |
+
Training...: 26%|████████████████████████████████████▊ | 90956/352766 [10:36<101:26:47, 1.39s/it]
|
114 |
+
|
115 |
+
|
116 |
+
|
117 |
+
|
118 |
+
|
119 |
+
Training...: 26%|█████████████████████████████████████▏ | 91008/352766 [10:56<66:45:48, 1.09it/s]
|
wandb/run-20210715_135329-1c6qs9ww/files/requirements.txt
ADDED
@@ -0,0 +1,94 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
absl-py==0.13.0
|
2 |
+
aiohttp==3.7.4.post0
|
3 |
+
astunparse==1.6.3
|
4 |
+
async-timeout==3.0.1
|
5 |
+
attrs==21.2.0
|
6 |
+
cachetools==4.2.2
|
7 |
+
certifi==2021.5.30
|
8 |
+
chardet==4.0.0
|
9 |
+
charset-normalizer==2.0.1
|
10 |
+
chex==0.0.8
|
11 |
+
click==8.0.1
|
12 |
+
configparser==5.0.2
|
13 |
+
cycler==0.10.0
|
14 |
+
datasets==1.9.1.dev0
|
15 |
+
dill==0.3.4
|
16 |
+
dm-tree==0.1.6
|
17 |
+
docker-pycreds==0.4.0
|
18 |
+
filelock==3.0.12
|
19 |
+
flatbuffers==1.12
|
20 |
+
flax==0.3.4
|
21 |
+
fsspec==2021.7.0
|
22 |
+
gast==0.4.0
|
23 |
+
gitdb==4.0.7
|
24 |
+
gitpython==3.1.18
|
25 |
+
google-auth-oauthlib==0.4.4
|
26 |
+
google-auth==1.32.1
|
27 |
+
google-pasta==0.2.0
|
28 |
+
grpcio==1.34.1
|
29 |
+
h5py==3.1.0
|
30 |
+
huggingface-hub==0.0.12
|
31 |
+
idna==3.2
|
32 |
+
install==1.3.4
|
33 |
+
jax==0.2.17
|
34 |
+
jaxlib==0.1.68
|
35 |
+
joblib==1.0.1
|
36 |
+
keras-nightly==2.5.0.dev2021032900
|
37 |
+
keras-preprocessing==1.1.2
|
38 |
+
kiwisolver==1.3.1
|
39 |
+
libtpu-nightly==0.1.dev20210615
|
40 |
+
markdown==3.3.4
|
41 |
+
matplotlib==3.4.2
|
42 |
+
msgpack==1.0.2
|
43 |
+
multidict==5.1.0
|
44 |
+
multiprocess==0.70.12.2
|
45 |
+
numpy==1.19.5
|
46 |
+
oauthlib==3.1.1
|
47 |
+
opt-einsum==3.3.0
|
48 |
+
optax==0.0.9
|
49 |
+
packaging==21.0
|
50 |
+
pandas==1.3.0
|
51 |
+
pathtools==0.1.2
|
52 |
+
pillow==8.3.1
|
53 |
+
pip==20.0.2
|
54 |
+
pkg-resources==0.0.0
|
55 |
+
promise==2.3
|
56 |
+
protobuf==3.17.3
|
57 |
+
psutil==5.8.0
|
58 |
+
pyarrow==4.0.1
|
59 |
+
pyasn1-modules==0.2.8
|
60 |
+
pyasn1==0.4.8
|
61 |
+
pyparsing==2.4.7
|
62 |
+
python-dateutil==2.8.1
|
63 |
+
pytz==2021.1
|
64 |
+
pyyaml==5.4.1
|
65 |
+
regex==2021.7.6
|
66 |
+
requests-oauthlib==1.3.0
|
67 |
+
requests==2.26.0
|
68 |
+
rsa==4.7.2
|
69 |
+
sacremoses==0.0.45
|
70 |
+
scipy==1.7.0
|
71 |
+
sentry-sdk==1.3.0
|
72 |
+
setuptools==44.0.0
|
73 |
+
shortuuid==1.0.1
|
74 |
+
six==1.15.0
|
75 |
+
smmap==4.0.0
|
76 |
+
subprocess32==3.5.4
|
77 |
+
tensorboard-data-server==0.6.1
|
78 |
+
tensorboard-plugin-wit==1.8.0
|
79 |
+
tensorboard==2.5.0
|
80 |
+
tensorflow-estimator==2.5.0
|
81 |
+
tensorflow==2.5.0
|
82 |
+
termcolor==1.1.0
|
83 |
+
tokenizers==0.10.3
|
84 |
+
toolz==0.11.1
|
85 |
+
tqdm==4.61.2
|
86 |
+
transformers==4.9.0.dev0
|
87 |
+
typing-extensions==3.7.4.3
|
88 |
+
urllib3==1.26.6
|
89 |
+
wandb==0.10.33
|
90 |
+
werkzeug==2.0.1
|
91 |
+
wheel==0.36.2
|
92 |
+
wrapt==1.12.1
|
93 |
+
xxhash==2.0.2
|
94 |
+
yarl==1.6.3
|
wandb/run-20210715_135329-1c6qs9ww/files/wandb-metadata.json
ADDED
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29",
|
3 |
+
"python": "3.8.10",
|
4 |
+
"heartbeatAt": "2021-07-15T13:53:31.167145",
|
5 |
+
"startedAt": "2021-07-15T13:53:29.138276",
|
6 |
+
"docker": null,
|
7 |
+
"cpu_count": 96,
|
8 |
+
"cuda": null,
|
9 |
+
"args": [
|
10 |
+
"--push_to_hub",
|
11 |
+
"--output_dir=./",
|
12 |
+
"--model_type=big_bird",
|
13 |
+
"--config_name=./",
|
14 |
+
"--tokenizer_name=./",
|
15 |
+
"--max_seq_length=4096",
|
16 |
+
"--weight_decay=0.0095",
|
17 |
+
"--warmup_steps=10000",
|
18 |
+
"--overwrite_output_dir",
|
19 |
+
"--adam_beta1=0.9",
|
20 |
+
"--adam_beta2=0.98",
|
21 |
+
"--logging_steps=50",
|
22 |
+
"--eval_steps=2000",
|
23 |
+
"--num_train_epochs=2",
|
24 |
+
"--preprocessing_num_workers=96",
|
25 |
+
"--save_steps=10000",
|
26 |
+
"--learning_rate=3e-5",
|
27 |
+
"--per_device_train_batch_size=1",
|
28 |
+
"--per_device_eval_batch_size=1",
|
29 |
+
"--save_total_limit=5",
|
30 |
+
"--max_eval_samples=5000",
|
31 |
+
"--resume_from_checkpoint=./"
|
32 |
+
],
|
33 |
+
"state": "running",
|
34 |
+
"program": "./run_mlm_flax_no_accum.py",
|
35 |
+
"codePath": "run_mlm_flax_no_accum.py",
|
36 |
+
"git": {
|
37 |
+
"remote": "https://huggingface.co/flax-community/pino-roberta-base",
|
38 |
+
"commit": "b6814517994e86d862ddec99dda42b9dbea5b8b0"
|
39 |
+
},
|
40 |
+
"email": null,
|
41 |
+
"root": "/home/dat/pino-roberta-base",
|
42 |
+
"host": "t1v-n-f5c06ea1-w-0",
|
43 |
+
"username": "dat",
|
44 |
+
"executable": "/home/dat/pino/bin/python"
|
45 |
+
}
|
wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
{"training_step": 1000, "learning_rate": 2.650622627697885e-05, "train_loss": 2.3977699279785156, "_runtime": 1003, "_timestamp": 1626358212, "_step": 19}
|
wandb/run-20210715_135329-1c6qs9ww/logs/debug-internal.log
ADDED
@@ -0,0 +1,399 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
2021-07-15 13:53:29,847 INFO MainThread:693510 [internal.py:wandb_internal():88] W&B internal server running at pid: 693510, started at: 2021-07-15 13:53:29.847625
|
2 |
+
2021-07-15 13:53:29,850 INFO WriterThread:693510 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/run-1c6qs9ww.wandb
|
3 |
+
2021-07-15 13:53:29,850 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: check_version
|
4 |
+
2021-07-15 13:53:29,851 DEBUG SenderThread:693510 [sender.py:send():179] send: header
|
5 |
+
2021-07-15 13:53:29,851 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: check_version
|
6 |
+
2021-07-15 13:53:29,891 DEBUG SenderThread:693510 [sender.py:send():179] send: run
|
7 |
+
2021-07-15 13:53:30,065 INFO SenderThread:693510 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files
|
8 |
+
2021-07-15 13:53:30,065 INFO SenderThread:693510 [sender.py:_start_run_threads():716] run started: 1c6qs9ww with start time 1626357209
|
9 |
+
2021-07-15 13:53:30,065 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
|
10 |
+
2021-07-15 13:53:30,066 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
|
11 |
+
2021-07-15 13:53:30,066 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: run_start
|
12 |
+
2021-07-15 13:53:31,068 INFO Thread-8 :693510 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
|
13 |
+
2021-07-15 13:53:31,166 DEBUG HandlerThread:693510 [meta.py:__init__():39] meta init
|
14 |
+
2021-07-15 13:53:31,167 DEBUG HandlerThread:693510 [meta.py:__init__():53] meta init done
|
15 |
+
2021-07-15 13:53:31,167 DEBUG HandlerThread:693510 [meta.py:probe():210] probe
|
16 |
+
2021-07-15 13:53:31,168 DEBUG HandlerThread:693510 [meta.py:_setup_git():200] setup git
|
17 |
+
2021-07-15 13:53:31,198 DEBUG HandlerThread:693510 [meta.py:_setup_git():207] setup git done
|
18 |
+
2021-07-15 13:53:31,198 DEBUG HandlerThread:693510 [meta.py:_save_pip():57] save pip
|
19 |
+
2021-07-15 13:53:31,199 DEBUG HandlerThread:693510 [meta.py:_save_pip():71] save pip done
|
20 |
+
2021-07-15 13:53:31,199 DEBUG HandlerThread:693510 [meta.py:probe():252] probe done
|
21 |
+
2021-07-15 13:53:31,202 DEBUG SenderThread:693510 [sender.py:send():179] send: files
|
22 |
+
2021-07-15 13:53:31,202 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now
|
23 |
+
2021-07-15 13:53:31,210 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
24 |
+
2021-07-15 13:53:31,211 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
25 |
+
2021-07-15 13:53:31,339 DEBUG SenderThread:693510 [sender.py:send():179] send: config
|
26 |
+
2021-07-15 13:53:31,340 DEBUG SenderThread:693510 [sender.py:send():179] send: config
|
27 |
+
2021-07-15 13:53:31,340 DEBUG SenderThread:693510 [sender.py:send():179] send: config
|
28 |
+
2021-07-15 13:53:31,710 INFO Thread-11 :693510 [upload_job.py:push():137] Uploaded file /tmp/tmpbh2gacmiwandb/3ahj6nuh-wandb-metadata.json
|
29 |
+
2021-07-15 13:53:32,067 INFO Thread-8 :693510 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/requirements.txt
|
30 |
+
2021-07-15 13:53:32,067 INFO Thread-8 :693510 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-metadata.json
|
31 |
+
2021-07-15 13:53:32,068 INFO Thread-8 :693510 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
32 |
+
2021-07-15 13:53:46,073 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
33 |
+
2021-07-15 13:53:46,399 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
34 |
+
2021-07-15 13:53:46,399 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
35 |
+
2021-07-15 13:53:49,074 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
36 |
+
2021-07-15 13:53:59,251 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
37 |
+
2021-07-15 13:54:01,078 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/config.yaml
|
38 |
+
2021-07-15 13:54:01,586 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
39 |
+
2021-07-15 13:54:01,587 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
40 |
+
2021-07-15 13:54:16,718 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
41 |
+
2021-07-15 13:54:16,719 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
42 |
+
2021-07-15 13:54:29,328 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
43 |
+
2021-07-15 13:54:31,849 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
44 |
+
2021-07-15 13:54:31,850 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
45 |
+
2021-07-15 13:54:46,986 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
46 |
+
2021-07-15 13:54:46,986 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
47 |
+
2021-07-15 13:54:59,405 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
48 |
+
2021-07-15 13:55:02,118 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
49 |
+
2021-07-15 13:55:02,118 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
50 |
+
2021-07-15 13:55:17,248 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
51 |
+
2021-07-15 13:55:17,249 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
52 |
+
2021-07-15 13:55:29,470 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
53 |
+
2021-07-15 13:55:32,381 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
54 |
+
2021-07-15 13:55:32,381 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
55 |
+
2021-07-15 13:55:47,513 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
56 |
+
2021-07-15 13:55:47,513 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
57 |
+
2021-07-15 13:55:59,545 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
58 |
+
2021-07-15 13:56:02,648 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
59 |
+
2021-07-15 13:56:02,648 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
60 |
+
2021-07-15 13:56:17,780 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
61 |
+
2021-07-15 13:56:17,780 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
62 |
+
2021-07-15 13:56:29,613 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
63 |
+
2021-07-15 13:56:32,912 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
64 |
+
2021-07-15 13:56:32,912 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
65 |
+
2021-07-15 13:56:48,044 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
66 |
+
2021-07-15 13:56:48,044 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
67 |
+
2021-07-15 13:56:59,677 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
68 |
+
2021-07-15 13:57:03,179 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
69 |
+
2021-07-15 13:57:03,179 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
70 |
+
2021-07-15 13:57:18,310 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
71 |
+
2021-07-15 13:57:18,311 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
72 |
+
2021-07-15 13:57:29,744 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
73 |
+
2021-07-15 13:57:33,446 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
74 |
+
2021-07-15 13:57:33,446 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
75 |
+
2021-07-15 13:57:48,577 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
76 |
+
2021-07-15 13:57:48,577 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
77 |
+
2021-07-15 13:57:59,820 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
78 |
+
2021-07-15 13:58:03,712 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
79 |
+
2021-07-15 13:58:03,712 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
80 |
+
2021-07-15 13:58:18,841 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
81 |
+
2021-07-15 13:58:18,841 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
82 |
+
2021-07-15 13:58:29,899 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
83 |
+
2021-07-15 13:58:33,972 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
84 |
+
2021-07-15 13:58:33,972 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
85 |
+
2021-07-15 13:58:49,102 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
86 |
+
2021-07-15 13:58:49,102 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
87 |
+
2021-07-15 13:58:59,963 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
88 |
+
2021-07-15 13:59:04,236 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
89 |
+
2021-07-15 13:59:04,236 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
90 |
+
2021-07-15 13:59:17,202 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
91 |
+
2021-07-15 13:59:19,407 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
92 |
+
2021-07-15 13:59:19,408 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
93 |
+
2021-07-15 13:59:30,030 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
94 |
+
2021-07-15 13:59:34,559 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
95 |
+
2021-07-15 13:59:34,559 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
96 |
+
2021-07-15 13:59:49,711 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
97 |
+
2021-07-15 13:59:49,712 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
98 |
+
2021-07-15 14:00:00,102 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
99 |
+
2021-07-15 14:00:04,842 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
100 |
+
2021-07-15 14:00:04,842 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
101 |
+
2021-07-15 14:00:19,976 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
102 |
+
2021-07-15 14:00:19,976 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
103 |
+
2021-07-15 14:00:30,175 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
104 |
+
2021-07-15 14:00:35,107 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
105 |
+
2021-07-15 14:00:35,107 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
106 |
+
2021-07-15 14:00:50,245 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
107 |
+
2021-07-15 14:00:50,246 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
108 |
+
2021-07-15 14:01:00,254 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
109 |
+
2021-07-15 14:01:05,377 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
110 |
+
2021-07-15 14:01:05,378 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
111 |
+
2021-07-15 14:01:19,253 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
112 |
+
2021-07-15 14:01:20,513 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
113 |
+
2021-07-15 14:01:20,513 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
114 |
+
2021-07-15 14:01:30,331 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
115 |
+
2021-07-15 14:01:35,686 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
116 |
+
2021-07-15 14:01:35,686 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
117 |
+
2021-07-15 14:01:50,838 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
118 |
+
2021-07-15 14:01:50,838 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
119 |
+
2021-07-15 14:02:00,415 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
120 |
+
2021-07-15 14:02:05,977 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
121 |
+
2021-07-15 14:02:05,977 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
122 |
+
2021-07-15 14:02:21,110 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
123 |
+
2021-07-15 14:02:21,110 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
124 |
+
2021-07-15 14:02:30,499 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
125 |
+
2021-07-15 14:02:36,243 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
126 |
+
2021-07-15 14:02:36,243 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
127 |
+
2021-07-15 14:02:51,378 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
128 |
+
2021-07-15 14:02:51,378 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
129 |
+
2021-07-15 14:03:00,580 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
130 |
+
2021-07-15 14:03:06,510 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
131 |
+
2021-07-15 14:03:06,511 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
132 |
+
2021-07-15 14:03:21,645 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
133 |
+
2021-07-15 14:03:21,645 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
134 |
+
2021-07-15 14:03:30,688 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
135 |
+
2021-07-15 14:03:34,306 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
136 |
+
2021-07-15 14:03:36,307 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
137 |
+
2021-07-15 14:03:36,775 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
138 |
+
2021-07-15 14:03:36,775 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
139 |
+
2021-07-15 14:03:38,307 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
140 |
+
2021-07-15 14:03:40,308 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
141 |
+
2021-07-15 14:03:51,638 DEBUG SenderThread:693510 [sender.py:send():179] send: history
|
142 |
+
2021-07-15 14:03:51,638 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
|
143 |
+
2021-07-15 14:03:51,639 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
|
144 |
+
2021-07-15 14:03:51,934 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
145 |
+
2021-07-15 14:03:51,934 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
146 |
+
2021-07-15 14:03:52,313 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
|
147 |
+
2021-07-15 14:03:53,314 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
148 |
+
2021-07-15 14:03:54,314 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
149 |
+
2021-07-15 14:03:56,315 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
150 |
+
2021-07-15 14:03:58,316 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
151 |
+
2021-07-15 14:04:00,316 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
152 |
+
2021-07-15 14:04:00,764 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
153 |
+
2021-07-15 14:04:07,066 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
154 |
+
2021-07-15 14:04:07,066 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
155 |
+
2021-07-15 14:04:11,676 DEBUG SenderThread:693510 [sender.py:send():179] send: history
|
156 |
+
2021-07-15 14:04:11,676 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
|
157 |
+
2021-07-15 14:04:11,677 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
|
158 |
+
2021-07-15 14:04:12,321 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
|
159 |
+
2021-07-15 14:04:13,321 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
160 |
+
2021-07-15 14:04:14,322 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
161 |
+
2021-07-15 14:04:16,323 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
162 |
+
2021-07-15 14:04:18,324 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
163 |
+
2021-07-15 14:04:20,325 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
164 |
+
2021-07-15 14:04:22,195 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
165 |
+
2021-07-15 14:04:22,196 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
166 |
+
2021-07-15 14:04:30,839 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
167 |
+
2021-07-15 14:04:31,712 DEBUG SenderThread:693510 [sender.py:send():179] send: history
|
168 |
+
2021-07-15 14:04:31,712 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
|
169 |
+
2021-07-15 14:04:31,713 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
|
170 |
+
2021-07-15 14:04:32,330 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
|
171 |
+
2021-07-15 14:04:33,331 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
172 |
+
2021-07-15 14:04:34,331 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
173 |
+
2021-07-15 14:04:36,332 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
174 |
+
2021-07-15 14:04:37,327 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
175 |
+
2021-07-15 14:04:37,327 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
176 |
+
2021-07-15 14:04:38,333 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
177 |
+
2021-07-15 14:04:40,334 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
178 |
+
2021-07-15 14:04:51,749 DEBUG SenderThread:693510 [sender.py:send():179] send: history
|
179 |
+
2021-07-15 14:04:51,749 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
|
180 |
+
2021-07-15 14:04:51,749 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
|
181 |
+
2021-07-15 14:04:52,339 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
|
182 |
+
2021-07-15 14:04:52,479 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
183 |
+
2021-07-15 14:04:52,479 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
184 |
+
2021-07-15 14:04:53,339 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
185 |
+
2021-07-15 14:04:54,340 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
186 |
+
2021-07-15 14:04:56,340 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
187 |
+
2021-07-15 14:04:58,341 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
188 |
+
2021-07-15 14:05:00,342 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
189 |
+
2021-07-15 14:05:00,907 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
190 |
+
2021-07-15 14:05:07,647 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
191 |
+
2021-07-15 14:05:07,648 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
192 |
+
2021-07-15 14:05:11,789 DEBUG SenderThread:693510 [sender.py:send():179] send: history
|
193 |
+
2021-07-15 14:05:11,789 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
|
194 |
+
2021-07-15 14:05:11,789 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
|
195 |
+
2021-07-15 14:05:12,347 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
|
196 |
+
2021-07-15 14:05:13,348 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
197 |
+
2021-07-15 14:05:14,348 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
198 |
+
2021-07-15 14:05:16,349 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
199 |
+
2021-07-15 14:05:18,350 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
200 |
+
2021-07-15 14:05:20,351 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
201 |
+
2021-07-15 14:05:22,781 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
202 |
+
2021-07-15 14:05:22,782 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
203 |
+
2021-07-15 14:05:30,976 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
204 |
+
2021-07-15 14:05:31,824 DEBUG SenderThread:693510 [sender.py:send():179] send: history
|
205 |
+
2021-07-15 14:05:31,824 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
|
206 |
+
2021-07-15 14:05:31,825 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
|
207 |
+
2021-07-15 14:05:32,355 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
|
208 |
+
2021-07-15 14:05:32,356 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
209 |
+
2021-07-15 14:05:33,356 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
210 |
+
2021-07-15 14:05:34,356 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
211 |
+
2021-07-15 14:05:36,357 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
212 |
+
2021-07-15 14:05:37,916 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
213 |
+
2021-07-15 14:05:37,916 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
214 |
+
2021-07-15 14:05:38,358 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
215 |
+
2021-07-15 14:05:40,359 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
216 |
+
2021-07-15 14:05:51,866 DEBUG SenderThread:693510 [sender.py:send():179] send: history
|
217 |
+
2021-07-15 14:05:51,866 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
|
218 |
+
2021-07-15 14:05:51,866 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
|
219 |
+
2021-07-15 14:05:52,364 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
|
220 |
+
2021-07-15 14:05:53,068 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
221 |
+
2021-07-15 14:05:53,068 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
222 |
+
2021-07-15 14:05:53,364 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
223 |
+
2021-07-15 14:05:54,365 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
224 |
+
2021-07-15 14:05:56,366 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
225 |
+
2021-07-15 14:05:58,367 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
226 |
+
2021-07-15 14:06:00,368 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
227 |
+
2021-07-15 14:06:01,051 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
228 |
+
2021-07-15 14:06:08,203 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
229 |
+
2021-07-15 14:06:08,203 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
230 |
+
2021-07-15 14:06:11,904 DEBUG SenderThread:693510 [sender.py:send():179] send: history
|
231 |
+
2021-07-15 14:06:11,904 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
|
232 |
+
2021-07-15 14:06:11,904 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
|
233 |
+
2021-07-15 14:06:12,372 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
|
234 |
+
2021-07-15 14:06:13,373 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
235 |
+
2021-07-15 14:06:14,373 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
236 |
+
2021-07-15 14:06:16,374 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
237 |
+
2021-07-15 14:06:18,375 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
238 |
+
2021-07-15 14:06:20,376 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
239 |
+
2021-07-15 14:06:23,336 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
240 |
+
2021-07-15 14:06:23,336 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
241 |
+
2021-07-15 14:06:31,124 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
242 |
+
2021-07-15 14:06:31,942 DEBUG SenderThread:693510 [sender.py:send():179] send: history
|
243 |
+
2021-07-15 14:06:31,943 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
|
244 |
+
2021-07-15 14:06:31,943 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
|
245 |
+
2021-07-15 14:06:32,381 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
|
246 |
+
2021-07-15 14:06:33,382 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
247 |
+
2021-07-15 14:06:34,382 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
248 |
+
2021-07-15 14:06:36,383 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
249 |
+
2021-07-15 14:06:38,384 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
250 |
+
2021-07-15 14:06:38,467 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
251 |
+
2021-07-15 14:06:38,468 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
252 |
+
2021-07-15 14:06:40,385 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
253 |
+
2021-07-15 14:06:51,987 DEBUG SenderThread:693510 [sender.py:send():179] send: history
|
254 |
+
2021-07-15 14:06:51,987 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
|
255 |
+
2021-07-15 14:06:51,988 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
|
256 |
+
2021-07-15 14:06:52,389 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
|
257 |
+
2021-07-15 14:06:53,390 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
258 |
+
2021-07-15 14:06:53,648 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
259 |
+
2021-07-15 14:06:53,648 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
260 |
+
2021-07-15 14:06:54,390 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
261 |
+
2021-07-15 14:06:56,391 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
262 |
+
2021-07-15 14:06:58,392 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
263 |
+
2021-07-15 14:07:00,393 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
264 |
+
2021-07-15 14:07:01,197 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
265 |
+
2021-07-15 14:07:08,814 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
266 |
+
2021-07-15 14:07:08,814 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
267 |
+
2021-07-15 14:07:12,024 DEBUG SenderThread:693510 [sender.py:send():179] send: history
|
268 |
+
2021-07-15 14:07:12,025 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
|
269 |
+
2021-07-15 14:07:12,025 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
|
270 |
+
2021-07-15 14:07:12,397 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
|
271 |
+
2021-07-15 14:07:12,397 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
272 |
+
2021-07-15 14:07:13,398 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
273 |
+
2021-07-15 14:07:14,398 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
274 |
+
2021-07-15 14:07:16,399 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
275 |
+
2021-07-15 14:07:18,400 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
276 |
+
2021-07-15 14:07:20,400 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
277 |
+
2021-07-15 14:07:23,978 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
278 |
+
2021-07-15 14:07:23,978 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
279 |
+
2021-07-15 14:07:31,275 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
280 |
+
2021-07-15 14:07:32,060 DEBUG SenderThread:693510 [sender.py:send():179] send: history
|
281 |
+
2021-07-15 14:07:32,060 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
|
282 |
+
2021-07-15 14:07:32,061 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
|
283 |
+
2021-07-15 14:07:32,406 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
|
284 |
+
2021-07-15 14:07:32,406 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
285 |
+
2021-07-15 14:07:33,406 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
286 |
+
2021-07-15 14:07:34,406 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
287 |
+
2021-07-15 14:07:36,407 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
288 |
+
2021-07-15 14:07:38,408 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
289 |
+
2021-07-15 14:07:39,111 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
290 |
+
2021-07-15 14:07:39,111 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
291 |
+
2021-07-15 14:07:40,409 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
292 |
+
2021-07-15 14:07:52,095 DEBUG SenderThread:693510 [sender.py:send():179] send: history
|
293 |
+
2021-07-15 14:07:52,095 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
|
294 |
+
2021-07-15 14:07:52,095 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
|
295 |
+
2021-07-15 14:07:52,414 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
|
296 |
+
2021-07-15 14:07:52,414 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
297 |
+
2021-07-15 14:07:54,262 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
298 |
+
2021-07-15 14:07:54,262 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
299 |
+
2021-07-15 14:07:54,414 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
300 |
+
2021-07-15 14:07:56,415 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
301 |
+
2021-07-15 14:07:58,416 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
302 |
+
2021-07-15 14:08:00,417 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
303 |
+
2021-07-15 14:08:01,410 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
304 |
+
2021-07-15 14:08:09,395 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
305 |
+
2021-07-15 14:08:09,395 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
306 |
+
2021-07-15 14:08:12,138 DEBUG SenderThread:693510 [sender.py:send():179] send: history
|
307 |
+
2021-07-15 14:08:12,138 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
|
308 |
+
2021-07-15 14:08:12,139 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
|
309 |
+
2021-07-15 14:08:12,422 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
|
310 |
+
2021-07-15 14:08:13,422 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
311 |
+
2021-07-15 14:08:14,423 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
312 |
+
2021-07-15 14:08:15,423 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
313 |
+
2021-07-15 14:08:17,424 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
314 |
+
2021-07-15 14:08:19,425 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
315 |
+
2021-07-15 14:08:21,426 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
316 |
+
2021-07-15 14:08:24,526 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
317 |
+
2021-07-15 14:08:24,527 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
318 |
+
2021-07-15 14:08:31,498 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
319 |
+
2021-07-15 14:08:32,177 DEBUG SenderThread:693510 [sender.py:send():179] send: history
|
320 |
+
2021-07-15 14:08:32,177 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
|
321 |
+
2021-07-15 14:08:32,177 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
|
322 |
+
2021-07-15 14:08:32,431 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
|
323 |
+
2021-07-15 14:08:33,431 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
324 |
+
2021-07-15 14:08:34,432 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
325 |
+
2021-07-15 14:08:35,432 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
326 |
+
2021-07-15 14:08:37,433 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
327 |
+
2021-07-15 14:08:39,434 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
328 |
+
2021-07-15 14:08:39,657 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
329 |
+
2021-07-15 14:08:39,657 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
330 |
+
2021-07-15 14:08:41,435 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
331 |
+
2021-07-15 14:08:52,218 DEBUG SenderThread:693510 [sender.py:send():179] send: history
|
332 |
+
2021-07-15 14:08:52,218 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
|
333 |
+
2021-07-15 14:08:52,218 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
|
334 |
+
2021-07-15 14:08:52,439 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
|
335 |
+
2021-07-15 14:08:53,440 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
336 |
+
2021-07-15 14:08:54,440 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
337 |
+
2021-07-15 14:08:54,790 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
338 |
+
2021-07-15 14:08:54,790 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
339 |
+
2021-07-15 14:08:55,441 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
340 |
+
2021-07-15 14:08:57,442 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
341 |
+
2021-07-15 14:08:59,443 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
342 |
+
2021-07-15 14:09:01,444 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
343 |
+
2021-07-15 14:09:01,574 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
344 |
+
2021-07-15 14:09:09,928 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
345 |
+
2021-07-15 14:09:09,928 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
346 |
+
2021-07-15 14:09:12,253 DEBUG SenderThread:693510 [sender.py:send():179] send: history
|
347 |
+
2021-07-15 14:09:12,254 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
|
348 |
+
2021-07-15 14:09:12,254 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
|
349 |
+
2021-07-15 14:09:12,448 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
|
350 |
+
2021-07-15 14:09:13,449 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
351 |
+
2021-07-15 14:09:14,449 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
352 |
+
2021-07-15 14:09:15,450 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
353 |
+
2021-07-15 14:09:17,451 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
354 |
+
2021-07-15 14:09:19,452 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
355 |
+
2021-07-15 14:09:21,453 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
356 |
+
2021-07-15 14:09:25,058 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
357 |
+
2021-07-15 14:09:25,058 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
358 |
+
2021-07-15 14:09:31,644 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
359 |
+
2021-07-15 14:09:32,292 DEBUG SenderThread:693510 [sender.py:send():179] send: history
|
360 |
+
2021-07-15 14:09:32,292 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
|
361 |
+
2021-07-15 14:09:32,293 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
|
362 |
+
2021-07-15 14:09:32,458 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
|
363 |
+
2021-07-15 14:09:33,458 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
364 |
+
2021-07-15 14:09:34,459 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
365 |
+
2021-07-15 14:09:35,459 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
366 |
+
2021-07-15 14:09:37,460 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
367 |
+
2021-07-15 14:09:39,461 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
368 |
+
2021-07-15 14:09:40,191 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
369 |
+
2021-07-15 14:09:40,192 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
370 |
+
2021-07-15 14:09:41,462 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
371 |
+
2021-07-15 14:09:52,334 DEBUG SenderThread:693510 [sender.py:send():179] send: history
|
372 |
+
2021-07-15 14:09:52,334 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
|
373 |
+
2021-07-15 14:09:52,335 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
|
374 |
+
2021-07-15 14:09:52,466 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
|
375 |
+
2021-07-15 14:09:53,466 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
376 |
+
2021-07-15 14:09:54,467 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
377 |
+
2021-07-15 14:09:55,325 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
378 |
+
2021-07-15 14:09:55,325 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
379 |
+
2021-07-15 14:09:55,467 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
380 |
+
2021-07-15 14:09:57,468 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
381 |
+
2021-07-15 14:09:59,469 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
382 |
+
2021-07-15 14:10:01,469 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
383 |
+
2021-07-15 14:10:01,719 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
|
384 |
+
2021-07-15 14:10:10,458 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
|
385 |
+
2021-07-15 14:10:10,458 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
|
386 |
+
2021-07-15 14:10:12,382 DEBUG SenderThread:693510 [sender.py:send():179] send: history
|
387 |
+
2021-07-15 14:10:12,383 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
|
388 |
+
2021-07-15 14:10:12,383 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
|
389 |
+
2021-07-15 14:10:12,474 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
|
390 |
+
2021-07-15 14:10:13,474 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
391 |
+
2021-07-15 14:10:14,354 WARNING MainThread:693510 [internal.py:wandb_internal():147] Internal process interrupt: 1
|
392 |
+
2021-07-15 14:10:14,475 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
|
393 |
+
2021-07-15 14:10:14,571 WARNING MainThread:693510 [internal.py:wandb_internal():147] Internal process interrupt: 2
|
394 |
+
2021-07-15 14:10:14,571 ERROR MainThread:693510 [internal.py:wandb_internal():150] Internal process interrupted.
|
395 |
+
2021-07-15 14:10:14,911 INFO SenderThread:693510 [sender.py:finish():945] shutting down sender
|
396 |
+
2021-07-15 14:10:14,912 INFO SenderThread:693510 [dir_watcher.py:finish():282] shutting down directory watcher
|
397 |
+
2021-07-15 14:10:14,912 INFO WriterThread:693510 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/run-1c6qs9ww.wandb
|
398 |
+
2021-07-15 14:10:14,912 INFO HandlerThread:693510 [handler.py:finish():638] shutting down handler
|
399 |
+
2021-07-15 14:10:15,102 INFO MainThread:693510 [internal.py:handle_exit():78] Internal process exited
|
wandb/run-20210715_135329-1c6qs9ww/logs/debug.log
ADDED
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
2021-07-15 13:53:29,139 INFO MainThread:692253 [wandb_setup.py:_flush():69] setting env: {}
|
2 |
+
2021-07-15 13:53:29,140 INFO MainThread:692253 [wandb_setup.py:_flush():69] setting login settings: {}
|
3 |
+
2021-07-15 13:53:29,140 INFO MainThread:692253 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/logs/debug.log
|
4 |
+
2021-07-15 13:53:29,140 INFO MainThread:692253 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/logs/debug-internal.log
|
5 |
+
2021-07-15 13:53:29,140 INFO MainThread:692253 [wandb_init.py:init():370] calling init triggers
|
6 |
+
2021-07-15 13:53:29,140 INFO MainThread:692253 [wandb_init.py:init():375] wandb.init called with sweep_config: {}
|
7 |
+
config: {}
|
8 |
+
2021-07-15 13:53:29,140 INFO MainThread:692253 [wandb_init.py:init():419] starting backend
|
9 |
+
2021-07-15 13:53:29,140 INFO MainThread:692253 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
|
10 |
+
2021-07-15 13:53:29,188 INFO MainThread:692253 [backend.py:ensure_launched():135] starting backend process...
|
11 |
+
2021-07-15 13:53:29,256 INFO MainThread:692253 [backend.py:ensure_launched():139] started backend process with pid: 693510
|
12 |
+
2021-07-15 13:53:29,257 INFO MainThread:692253 [wandb_init.py:init():424] backend started and connected
|
13 |
+
2021-07-15 13:53:29,260 INFO MainThread:692253 [wandb_init.py:init():472] updated telemetry
|
14 |
+
2021-07-15 13:53:29,261 INFO MainThread:692253 [wandb_init.py:init():491] communicating current version
|
15 |
+
2021-07-15 13:53:29,890 INFO MainThread:692253 [wandb_init.py:init():496] got version response
|
16 |
+
2021-07-15 13:53:29,890 INFO MainThread:692253 [wandb_init.py:init():504] communicating run to backend with 30 second timeout
|
17 |
+
2021-07-15 13:53:30,065 INFO MainThread:692253 [wandb_init.py:init():529] starting run threads in backend
|
18 |
+
2021-07-15 13:53:31,205 INFO MainThread:692253 [wandb_run.py:_console_start():1623] atexit reg
|
19 |
+
2021-07-15 13:53:31,206 INFO MainThread:692253 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT
|
20 |
+
2021-07-15 13:53:31,206 INFO MainThread:692253 [wandb_run.py:_redirect():1502] Redirecting console.
|
21 |
+
2021-07-15 13:53:31,208 INFO MainThread:692253 [wandb_run.py:_redirect():1558] Redirects installed.
|
22 |
+
2021-07-15 13:53:31,208 INFO MainThread:692253 [wandb_init.py:init():554] run started, returning control to user process
|
23 |
+
2021-07-15 13:53:31,215 INFO MainThread:692253 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 3e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 2.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul15_13-53-21_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 50, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 10000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 2000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': './', 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''}
|
24 |
+
2021-07-15 13:53:31,216 INFO MainThread:692253 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'}
|
25 |
+
2021-07-15 13:53:31,218 INFO MainThread:692253 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 5000}
|
26 |
+
2021-07-15 14:10:14,362 INFO MainThread:692253 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 255
|
27 |
+
2021-07-15 14:10:14,363 INFO MainThread:692253 [wandb_run.py:_restore():1565] restore
|
wandb/run-20210715_135329-1c6qs9ww/run-1c6qs9ww.wandb
ADDED
Binary file (46 kB). View file
|
|
wandb/run-20210715_141143-1vry5yso/files/config.yaml
ADDED
@@ -0,0 +1,301 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
wandb_version: 1
|
2 |
+
|
3 |
+
_wandb:
|
4 |
+
desc: null
|
5 |
+
value:
|
6 |
+
cli_version: 0.10.33
|
7 |
+
framework: huggingface
|
8 |
+
huggingface_version: 4.9.0.dev0
|
9 |
+
is_jupyter_run: false
|
10 |
+
is_kaggle_kernel: false
|
11 |
+
python_version: 3.8.10
|
12 |
+
t:
|
13 |
+
1:
|
14 |
+
- 3
|
15 |
+
- 11
|
16 |
+
4: 3.8.10
|
17 |
+
5: 0.10.33
|
18 |
+
6: 4.9.0.dev0
|
19 |
+
8:
|
20 |
+
- 5
|
21 |
+
adafactor:
|
22 |
+
desc: null
|
23 |
+
value: false
|
24 |
+
adam_beta1:
|
25 |
+
desc: null
|
26 |
+
value: 0.9
|
27 |
+
adam_beta2:
|
28 |
+
desc: null
|
29 |
+
value: 0.98
|
30 |
+
adam_epsilon:
|
31 |
+
desc: null
|
32 |
+
value: 1.0e-08
|
33 |
+
cache_dir:
|
34 |
+
desc: null
|
35 |
+
value: null
|
36 |
+
config_name:
|
37 |
+
desc: null
|
38 |
+
value: ./
|
39 |
+
dataloader_drop_last:
|
40 |
+
desc: null
|
41 |
+
value: false
|
42 |
+
dataloader_num_workers:
|
43 |
+
desc: null
|
44 |
+
value: 0
|
45 |
+
dataloader_pin_memory:
|
46 |
+
desc: null
|
47 |
+
value: true
|
48 |
+
dataset_config_name:
|
49 |
+
desc: null
|
50 |
+
value: null
|
51 |
+
dataset_name:
|
52 |
+
desc: null
|
53 |
+
value: null
|
54 |
+
ddp_find_unused_parameters:
|
55 |
+
desc: null
|
56 |
+
value: null
|
57 |
+
debug:
|
58 |
+
desc: null
|
59 |
+
value: []
|
60 |
+
deepspeed:
|
61 |
+
desc: null
|
62 |
+
value: null
|
63 |
+
disable_tqdm:
|
64 |
+
desc: null
|
65 |
+
value: false
|
66 |
+
do_eval:
|
67 |
+
desc: null
|
68 |
+
value: false
|
69 |
+
do_predict:
|
70 |
+
desc: null
|
71 |
+
value: false
|
72 |
+
do_train:
|
73 |
+
desc: null
|
74 |
+
value: false
|
75 |
+
dtype:
|
76 |
+
desc: null
|
77 |
+
value: float32
|
78 |
+
eval_accumulation_steps:
|
79 |
+
desc: null
|
80 |
+
value: null
|
81 |
+
eval_steps:
|
82 |
+
desc: null
|
83 |
+
value: 2000
|
84 |
+
evaluation_strategy:
|
85 |
+
desc: null
|
86 |
+
value: IntervalStrategy.NO
|
87 |
+
fp16:
|
88 |
+
desc: null
|
89 |
+
value: false
|
90 |
+
fp16_backend:
|
91 |
+
desc: null
|
92 |
+
value: auto
|
93 |
+
fp16_full_eval:
|
94 |
+
desc: null
|
95 |
+
value: false
|
96 |
+
fp16_opt_level:
|
97 |
+
desc: null
|
98 |
+
value: O1
|
99 |
+
gradient_accumulation_steps:
|
100 |
+
desc: null
|
101 |
+
value: 1
|
102 |
+
greater_is_better:
|
103 |
+
desc: null
|
104 |
+
value: null
|
105 |
+
group_by_length:
|
106 |
+
desc: null
|
107 |
+
value: false
|
108 |
+
ignore_data_skip:
|
109 |
+
desc: null
|
110 |
+
value: false
|
111 |
+
label_names:
|
112 |
+
desc: null
|
113 |
+
value: null
|
114 |
+
label_smoothing_factor:
|
115 |
+
desc: null
|
116 |
+
value: 0.0
|
117 |
+
learning_rate:
|
118 |
+
desc: null
|
119 |
+
value: 3.0e-05
|
120 |
+
length_column_name:
|
121 |
+
desc: null
|
122 |
+
value: length
|
123 |
+
line_by_line:
|
124 |
+
desc: null
|
125 |
+
value: false
|
126 |
+
load_best_model_at_end:
|
127 |
+
desc: null
|
128 |
+
value: false
|
129 |
+
local_rank:
|
130 |
+
desc: null
|
131 |
+
value: -1
|
132 |
+
log_level:
|
133 |
+
desc: null
|
134 |
+
value: -1
|
135 |
+
log_level_replica:
|
136 |
+
desc: null
|
137 |
+
value: -1
|
138 |
+
log_on_each_node:
|
139 |
+
desc: null
|
140 |
+
value: true
|
141 |
+
logging_dir:
|
142 |
+
desc: null
|
143 |
+
value: ./runs/Jul15_14-11-35_t1v-n-f5c06ea1-w-0
|
144 |
+
logging_first_step:
|
145 |
+
desc: null
|
146 |
+
value: false
|
147 |
+
logging_steps:
|
148 |
+
desc: null
|
149 |
+
value: 50
|
150 |
+
logging_strategy:
|
151 |
+
desc: null
|
152 |
+
value: IntervalStrategy.STEPS
|
153 |
+
lr_scheduler_type:
|
154 |
+
desc: null
|
155 |
+
value: SchedulerType.LINEAR
|
156 |
+
max_eval_samples:
|
157 |
+
desc: null
|
158 |
+
value: 5000
|
159 |
+
max_grad_norm:
|
160 |
+
desc: null
|
161 |
+
value: 1.0
|
162 |
+
max_seq_length:
|
163 |
+
desc: null
|
164 |
+
value: 4096
|
165 |
+
max_steps:
|
166 |
+
desc: null
|
167 |
+
value: -1
|
168 |
+
metric_for_best_model:
|
169 |
+
desc: null
|
170 |
+
value: null
|
171 |
+
mlm_probability:
|
172 |
+
desc: null
|
173 |
+
value: 0.15
|
174 |
+
model_name_or_path:
|
175 |
+
desc: null
|
176 |
+
value: null
|
177 |
+
model_type:
|
178 |
+
desc: null
|
179 |
+
value: big_bird
|
180 |
+
mp_parameters:
|
181 |
+
desc: null
|
182 |
+
value: ''
|
183 |
+
no_cuda:
|
184 |
+
desc: null
|
185 |
+
value: false
|
186 |
+
num_train_epochs:
|
187 |
+
desc: null
|
188 |
+
value: 2.0
|
189 |
+
output_dir:
|
190 |
+
desc: null
|
191 |
+
value: ./
|
192 |
+
overwrite_cache:
|
193 |
+
desc: null
|
194 |
+
value: false
|
195 |
+
overwrite_output_dir:
|
196 |
+
desc: null
|
197 |
+
value: true
|
198 |
+
pad_to_max_length:
|
199 |
+
desc: null
|
200 |
+
value: false
|
201 |
+
past_index:
|
202 |
+
desc: null
|
203 |
+
value: -1
|
204 |
+
per_device_eval_batch_size:
|
205 |
+
desc: null
|
206 |
+
value: 1
|
207 |
+
per_device_train_batch_size:
|
208 |
+
desc: null
|
209 |
+
value: 1
|
210 |
+
per_gpu_eval_batch_size:
|
211 |
+
desc: null
|
212 |
+
value: null
|
213 |
+
per_gpu_train_batch_size:
|
214 |
+
desc: null
|
215 |
+
value: null
|
216 |
+
prediction_loss_only:
|
217 |
+
desc: null
|
218 |
+
value: false
|
219 |
+
preprocessing_num_workers:
|
220 |
+
desc: null
|
221 |
+
value: 96
|
222 |
+
push_to_hub:
|
223 |
+
desc: null
|
224 |
+
value: true
|
225 |
+
push_to_hub_model_id:
|
226 |
+
desc: null
|
227 |
+
value: ''
|
228 |
+
push_to_hub_organization:
|
229 |
+
desc: null
|
230 |
+
value: null
|
231 |
+
push_to_hub_token:
|
232 |
+
desc: null
|
233 |
+
value: null
|
234 |
+
remove_unused_columns:
|
235 |
+
desc: null
|
236 |
+
value: true
|
237 |
+
report_to:
|
238 |
+
desc: null
|
239 |
+
value:
|
240 |
+
- tensorboard
|
241 |
+
- wandb
|
242 |
+
resume_from_checkpoint:
|
243 |
+
desc: null
|
244 |
+
value: ./
|
245 |
+
run_name:
|
246 |
+
desc: null
|
247 |
+
value: ./
|
248 |
+
save_on_each_node:
|
249 |
+
desc: null
|
250 |
+
value: false
|
251 |
+
save_steps:
|
252 |
+
desc: null
|
253 |
+
value: 10000
|
254 |
+
save_strategy:
|
255 |
+
desc: null
|
256 |
+
value: IntervalStrategy.STEPS
|
257 |
+
save_total_limit:
|
258 |
+
desc: null
|
259 |
+
value: 5
|
260 |
+
seed:
|
261 |
+
desc: null
|
262 |
+
value: 42
|
263 |
+
sharded_ddp:
|
264 |
+
desc: null
|
265 |
+
value: []
|
266 |
+
skip_memory_metrics:
|
267 |
+
desc: null
|
268 |
+
value: true
|
269 |
+
tokenizer_name:
|
270 |
+
desc: null
|
271 |
+
value: ./
|
272 |
+
tpu_metrics_debug:
|
273 |
+
desc: null
|
274 |
+
value: false
|
275 |
+
tpu_num_cores:
|
276 |
+
desc: null
|
277 |
+
value: null
|
278 |
+
train_ref_file:
|
279 |
+
desc: null
|
280 |
+
value: null
|
281 |
+
use_fast_tokenizer:
|
282 |
+
desc: null
|
283 |
+
value: true
|
284 |
+
use_legacy_prediction_loop:
|
285 |
+
desc: null
|
286 |
+
value: false
|
287 |
+
validation_ref_file:
|
288 |
+
desc: null
|
289 |
+
value: null
|
290 |
+
validation_split_percentage:
|
291 |
+
desc: null
|
292 |
+
value: 5
|
293 |
+
warmup_ratio:
|
294 |
+
desc: null
|
295 |
+
value: 0.0
|
296 |
+
warmup_steps:
|
297 |
+
desc: null
|
298 |
+
value: 10000
|
299 |
+
weight_decay:
|
300 |
+
desc: null
|
301 |
+
value: 0.0095
|
wandb/run-20210715_141143-1vry5yso/files/output.log
ADDED
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
|
2 |
+
[14:11:58] - INFO - absl - Restoring checkpoint from ./checkpoint_90000
|
3 |
+
tcmalloc: large alloc 1530273792 bytes == 0x9aa8a000 @ 0x7f287c292680 0x7f287c2b3824 0x5b9a14 0x50b2ae 0x50cb1b 0x5a6f17 0x5f3010 0x56fd36 0x568d9a 0x5f5b33 0x56aadf 0x568d9a 0x68cdc7 0x67e161 0x67e1df 0x67e281 0x67e627 0x6b6e62 0x6b71ed 0x7f287c0a70b3 0x5f96de
|
4 |
+
/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:386: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code.
|
5 |
+
warnings.warn(
|
6 |
+
/home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:373: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code.
|
7 |
+
warnings.warn(
|
8 |
+
Epoch ... (1/2): 0%| | 0/2 [00:00<?, ?it/s]
|
9 |
+
|
10 |
+
Traceback (most recent call last):███████████████████▉ | 90002/352766 [02:07<9322:46:40, 127.73s/it]
|
11 |
+
File "/usr/lib/python3.8/threading.py", line 932, in _bootstrap_inner
|
12 |
+
self.run()
|
13 |
+
File "/usr/lib/python3.8/threading.py", line 870, in run
|
14 |
+
self._target(*self._args, **self._kwargs)
|
15 |
+
File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/wandb_run.py", line 183, in check_network_status
|
16 |
+
status_response = self._interface.communicate_network_status()
|
17 |
+
File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/interface/interface.py", line 755, in communicate_network_status
|
18 |
+
resp = self._communicate(req, timeout=timeout, local=True)
|
19 |
+
File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/interface/interface.py", line 545, in _communicate
|
20 |
+
return self._communicate_async(rec, local=local).get(timeout=timeout)
|
21 |
+
File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/interface/interface.py", line 550, in _communicate_async
|
22 |
+
raise Exception("The wandb backend process has shutdown")
|
23 |
+
Exception: The wandb backend process has shutdown
|
24 |
+
Exception in thread Thread-3:
|
25 |
+
Traceback (most recent call last):
|
26 |
+
File "/usr/lib/python3.8/threading.py", line 932, in _bootstrap_inner
|
27 |
+
self.run()
|
28 |
+
File "/usr/lib/python3.8/threading.py", line 870, in run
|
29 |
+
self._target(*self._args, **self._kwargs)
|
30 |
+
File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/wandb_run.py", line 201, in check_status
|
31 |
+
status_response = self._interface.communicate_stop_status()
|
32 |
+
File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/interface/interface.py", line 743, in communicate_stop_status
|
33 |
+
resp = self._communicate(req, timeout=timeout, local=True)
|
34 |
+
File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/interface/interface.py", line 545, in _communicate
|
35 |
+
return self._communicate_async(rec, local=local).get(timeout=timeout)
|
36 |
+
File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/interface/interface.py", line 550, in _communicate_async
|
37 |
+
raise Exception("The wandb backend process has shutdown")
|
38 |
+
Exception: The wandb backend process has shutdown
|
39 |
+
Training...: 26%|███████████████████████████████████▋ | 90002/352766 [04:22<19182:56:09, 262.82s/it]
|
40 |
+
Epoch ... (1/2): 0%| | 0/2 [09:51<?, ?it/s]
|
41 |
+
Traceback (most recent call last):
|
42 |
+
File "./run_mlm_flax_no_accum.py", line 701, in <module>
|
43 |
+
state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs)
|
44 |
+
File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 183, in reraise_with_filtered_traceback
|
45 |
+
return fun(*args, **kwargs)
|
46 |
+
File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/api.py", line 1669, in f_pmapped
|
47 |
+
out = pxla.xla_pmap(
|
48 |
+
File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1620, in bind
|
49 |
+
return call_bind(self, fun, *args, **params)
|
50 |
+
File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1551, in call_bind
|
51 |
+
outs = primitive.process(top_trace, fun, tracers, params)
|
52 |
+
File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1623, in process
|
53 |
+
return trace.process_map(self, fun, tracers, params)
|
54 |
+
File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 606, in process_call
|
55 |
+
return primitive.impl(f, *tracers, **params)
|
56 |
+
File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 624, in xla_pmap_impl
|
57 |
+
compiled_fun, fingerprint = parallel_callable(fun, backend, axis_name, axis_size,
|
58 |
+
File "/home/dat/pino/lib/python3.8/site-packages/jax/linear_util.py", line 262, in memoized_fun
|
59 |
+
ans = call(fun, *args)
|
60 |
+
File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 906, in parallel_callable
|
61 |
+
compiled = xla.backend_compile(backend, built, compile_options)
|
62 |
+
File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/xla.py", line 360, in backend_compile
|
63 |
+
return backend.compile(built_c, compile_options=options)
|
64 |
+
KeyboardInterrupt
|
wandb/run-20210715_141143-1vry5yso/files/requirements.txt
ADDED
@@ -0,0 +1,94 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
absl-py==0.13.0
|
2 |
+
aiohttp==3.7.4.post0
|
3 |
+
astunparse==1.6.3
|
4 |
+
async-timeout==3.0.1
|
5 |
+
attrs==21.2.0
|
6 |
+
cachetools==4.2.2
|
7 |
+
certifi==2021.5.30
|
8 |
+
chardet==4.0.0
|
9 |
+
charset-normalizer==2.0.1
|
10 |
+
chex==0.0.8
|
11 |
+
click==8.0.1
|
12 |
+
configparser==5.0.2
|
13 |
+
cycler==0.10.0
|
14 |
+
datasets==1.9.1.dev0
|
15 |
+
dill==0.3.4
|
16 |
+
dm-tree==0.1.6
|
17 |
+
docker-pycreds==0.4.0
|
18 |
+
filelock==3.0.12
|
19 |
+
flatbuffers==1.12
|
20 |
+
flax==0.3.4
|
21 |
+
fsspec==2021.7.0
|
22 |
+
gast==0.4.0
|
23 |
+
gitdb==4.0.7
|
24 |
+
gitpython==3.1.18
|
25 |
+
google-auth-oauthlib==0.4.4
|
26 |
+
google-auth==1.32.1
|
27 |
+
google-pasta==0.2.0
|
28 |
+
grpcio==1.34.1
|
29 |
+
h5py==3.1.0
|
30 |
+
huggingface-hub==0.0.12
|
31 |
+
idna==3.2
|
32 |
+
install==1.3.4
|
33 |
+
jax==0.2.17
|
34 |
+
jaxlib==0.1.68
|
35 |
+
joblib==1.0.1
|
36 |
+
keras-nightly==2.5.0.dev2021032900
|
37 |
+
keras-preprocessing==1.1.2
|
38 |
+
kiwisolver==1.3.1
|
39 |
+
libtpu-nightly==0.1.dev20210615
|
40 |
+
markdown==3.3.4
|
41 |
+
matplotlib==3.4.2
|
42 |
+
msgpack==1.0.2
|
43 |
+
multidict==5.1.0
|
44 |
+
multiprocess==0.70.12.2
|
45 |
+
numpy==1.19.5
|
46 |
+
oauthlib==3.1.1
|
47 |
+
opt-einsum==3.3.0
|
48 |
+
optax==0.0.9
|
49 |
+
packaging==21.0
|
50 |
+
pandas==1.3.0
|
51 |
+
pathtools==0.1.2
|
52 |
+
pillow==8.3.1
|
53 |
+
pip==20.0.2
|
54 |
+
pkg-resources==0.0.0
|
55 |
+
promise==2.3
|
56 |
+
protobuf==3.17.3
|
57 |
+
psutil==5.8.0
|
58 |
+
pyarrow==4.0.1
|
59 |
+
pyasn1-modules==0.2.8
|
60 |
+
pyasn1==0.4.8
|
61 |
+
pyparsing==2.4.7
|
62 |
+
python-dateutil==2.8.1
|
63 |
+
pytz==2021.1
|
64 |
+
pyyaml==5.4.1
|
65 |
+
regex==2021.7.6
|
66 |
+
requests-oauthlib==1.3.0
|
67 |
+
requests==2.26.0
|
68 |
+
rsa==4.7.2
|
69 |
+
sacremoses==0.0.45
|
70 |
+
scipy==1.7.0
|
71 |
+
sentry-sdk==1.3.0
|
72 |
+
setuptools==44.0.0
|
73 |
+
shortuuid==1.0.1
|
74 |
+
six==1.15.0
|
75 |
+
smmap==4.0.0
|
76 |
+
subprocess32==3.5.4
|
77 |
+
tensorboard-data-server==0.6.1
|
78 |
+
tensorboard-plugin-wit==1.8.0
|
79 |
+
tensorboard==2.5.0
|
80 |
+
tensorflow-estimator==2.5.0
|
81 |
+
tensorflow==2.5.0
|
82 |
+
termcolor==1.1.0
|
83 |
+
tokenizers==0.10.3
|
84 |
+
toolz==0.11.1
|
85 |
+
tqdm==4.61.2
|
86 |
+
transformers==4.9.0.dev0
|
87 |
+
typing-extensions==3.7.4.3
|
88 |
+
urllib3==1.26.6
|
89 |
+
wandb==0.10.33
|
90 |
+
werkzeug==2.0.1
|
91 |
+
wheel==0.36.2
|
92 |
+
wrapt==1.12.1
|
93 |
+
xxhash==2.0.2
|
94 |
+
yarl==1.6.3
|
wandb/run-20210715_141143-1vry5yso/files/wandb-metadata.json
ADDED
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29",
|
3 |
+
"python": "3.8.10",
|
4 |
+
"heartbeatAt": "2021-07-15T14:11:46.095724",
|
5 |
+
"startedAt": "2021-07-15T14:11:43.787979",
|
6 |
+
"docker": null,
|
7 |
+
"cpu_count": 96,
|
8 |
+
"cuda": null,
|
9 |
+
"args": [
|
10 |
+
"--push_to_hub",
|
11 |
+
"--output_dir=./",
|
12 |
+
"--model_type=big_bird",
|
13 |
+
"--config_name=./",
|
14 |
+
"--tokenizer_name=./",
|
15 |
+
"--max_seq_length=4096",
|
16 |
+
"--weight_decay=0.0095",
|
17 |
+
"--warmup_steps=10000",
|
18 |
+
"--overwrite_output_dir",
|
19 |
+
"--adam_beta1=0.9",
|
20 |
+
"--adam_beta2=0.98",
|
21 |
+
"--logging_steps=50",
|
22 |
+
"--eval_steps=2000",
|
23 |
+
"--num_train_epochs=2",
|
24 |
+
"--preprocessing_num_workers=96",
|
25 |
+
"--save_steps=10000",
|
26 |
+
"--learning_rate=3e-5",
|
27 |
+
"--per_device_train_batch_size=1",
|
28 |
+
"--per_device_eval_batch_size=1",
|
29 |
+
"--save_total_limit=5",
|
30 |
+
"--max_eval_samples=5000",
|
31 |
+
"--resume_from_checkpoint=./"
|
32 |
+
],
|
33 |
+
"state": "running",
|
34 |
+
"program": "./run_mlm_flax_no_accum.py",
|
35 |
+
"codePath": "run_mlm_flax_no_accum.py",
|
36 |
+
"git": {
|
37 |
+
"remote": "https://huggingface.co/flax-community/pino-roberta-base",
|
38 |
+
"commit": "b6814517994e86d862ddec99dda42b9dbea5b8b0"
|
39 |
+
},
|
40 |
+
"email": null,
|
41 |
+
"root": "/home/dat/pino-roberta-base",
|
42 |
+
"host": "t1v-n-f5c06ea1-w-0",
|
43 |
+
"username": "dat",
|
44 |
+
"executable": "/home/dat/pino/bin/python"
|
45 |
+
}
|
wandb/run-20210715_141143-1vry5yso/files/wandb-summary.json
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
{}
|
wandb/run-20210715_141143-1vry5yso/logs/debug-internal.log
ADDED
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
2021-07-15 14:11:44,509 INFO MainThread:696553 [internal.py:wandb_internal():88] W&B internal server running at pid: 696553, started at: 2021-07-15 14:11:44.509350
|
2 |
+
2021-07-15 14:11:44,511 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: check_version
|
3 |
+
2021-07-15 14:11:44,512 INFO WriterThread:696553 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210715_141143-1vry5yso/run-1vry5yso.wandb
|
4 |
+
2021-07-15 14:11:44,513 DEBUG SenderThread:696553 [sender.py:send():179] send: header
|
5 |
+
2021-07-15 14:11:44,513 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: check_version
|
6 |
+
2021-07-15 14:11:44,549 DEBUG SenderThread:696553 [sender.py:send():179] send: run
|
7 |
+
2021-07-15 14:11:44,551 INFO SenderThread:696553 [sender.py:_maybe_setup_resume():413] checking resume status for wandb/hf-flax-pino-roberta/1vry5yso
|
8 |
+
2021-07-15 14:11:44,848 INFO SenderThread:696553 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210715_141143-1vry5yso/files
|
9 |
+
2021-07-15 14:11:44,849 INFO SenderThread:696553 [sender.py:_start_run_threads():716] run started: 1vry5yso with start time 1626358303
|
10 |
+
2021-07-15 14:11:44,849 DEBUG SenderThread:696553 [sender.py:send():179] send: summary
|
11 |
+
2021-07-15 14:11:44,849 INFO SenderThread:696553 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
|
12 |
+
2021-07-15 14:11:44,850 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: run_start
|
13 |
+
2021-07-15 14:11:45,906 INFO Thread-8 :696553 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_141143-1vry5yso/files/wandb-summary.json
|
14 |
+
2021-07-15 14:11:46,095 DEBUG HandlerThread:696553 [meta.py:__init__():39] meta init
|
15 |
+
2021-07-15 14:11:46,095 DEBUG HandlerThread:696553 [meta.py:__init__():53] meta init done
|
16 |
+
2021-07-15 14:11:46,095 DEBUG HandlerThread:696553 [meta.py:probe():210] probe
|
17 |
+
2021-07-15 14:11:46,097 DEBUG HandlerThread:696553 [meta.py:_setup_git():200] setup git
|
18 |
+
2021-07-15 14:11:46,132 DEBUG HandlerThread:696553 [meta.py:_setup_git():207] setup git done
|
19 |
+
2021-07-15 14:11:46,132 DEBUG HandlerThread:696553 [meta.py:_save_pip():57] save pip
|
20 |
+
2021-07-15 14:11:46,132 DEBUG HandlerThread:696553 [meta.py:_save_pip():71] save pip done
|
21 |
+
2021-07-15 14:11:46,132 DEBUG HandlerThread:696553 [meta.py:probe():252] probe done
|
22 |
+
2021-07-15 14:11:46,136 DEBUG SenderThread:696553 [sender.py:send():179] send: files
|
23 |
+
2021-07-15 14:11:46,136 INFO SenderThread:696553 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now
|
24 |
+
2021-07-15 14:11:46,144 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
25 |
+
2021-07-15 14:11:46,144 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
26 |
+
2021-07-15 14:11:46,273 DEBUG SenderThread:696553 [sender.py:send():179] send: config
|
27 |
+
2021-07-15 14:11:46,274 DEBUG SenderThread:696553 [sender.py:send():179] send: config
|
28 |
+
2021-07-15 14:11:46,274 DEBUG SenderThread:696553 [sender.py:send():179] send: config
|
29 |
+
2021-07-15 14:11:46,589 INFO Thread-11 :696553 [upload_job.py:push():137] Uploaded file /tmp/tmp6ba_u0b3wandb/11b0hrsr-wandb-metadata.json
|
30 |
+
2021-07-15 14:11:46,906 INFO Thread-8 :696553 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_141143-1vry5yso/files/requirements.txt
|
31 |
+
2021-07-15 14:11:46,906 INFO Thread-8 :696553 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_141143-1vry5yso/files/wandb-metadata.json
|
32 |
+
2021-07-15 14:11:46,907 INFO Thread-8 :696553 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_141143-1vry5yso/files/output.log
|
33 |
+
2021-07-15 14:12:01,364 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
34 |
+
2021-07-15 14:12:01,364 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
35 |
+
2021-07-15 14:12:01,912 INFO Thread-8 :696553 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_141143-1vry5yso/files/output.log
|
36 |
+
2021-07-15 14:12:03,914 INFO Thread-8 :696553 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_141143-1vry5yso/files/output.log
|
37 |
+
2021-07-15 14:12:14,183 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
|
38 |
+
2021-07-15 14:12:15,919 INFO Thread-8 :696553 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_141143-1vry5yso/files/config.yaml
|
39 |
+
2021-07-15 14:12:16,527 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
40 |
+
2021-07-15 14:12:16,527 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
41 |
+
2021-07-15 14:12:31,659 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
42 |
+
2021-07-15 14:12:31,660 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
43 |
+
2021-07-15 14:12:44,260 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
|
44 |
+
2021-07-15 14:12:46,795 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
45 |
+
2021-07-15 14:12:46,795 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
46 |
+
2021-07-15 14:13:01,927 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
47 |
+
2021-07-15 14:13:01,928 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
48 |
+
2021-07-15 14:13:14,337 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
|
49 |
+
2021-07-15 14:13:17,060 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
50 |
+
2021-07-15 14:13:17,060 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
51 |
+
2021-07-15 14:13:32,199 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
52 |
+
2021-07-15 14:13:32,199 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
53 |
+
2021-07-15 14:13:44,410 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
|
54 |
+
2021-07-15 14:13:47,335 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
55 |
+
2021-07-15 14:13:47,335 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
56 |
+
2021-07-15 14:14:02,468 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
57 |
+
2021-07-15 14:14:02,468 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
58 |
+
2021-07-15 14:14:14,489 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
|
59 |
+
2021-07-15 14:14:17,599 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
60 |
+
2021-07-15 14:14:17,600 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
61 |
+
2021-07-15 14:14:32,741 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
62 |
+
2021-07-15 14:14:32,742 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
63 |
+
2021-07-15 14:14:44,568 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
|
64 |
+
2021-07-15 14:14:47,874 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
65 |
+
2021-07-15 14:14:47,875 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
66 |
+
2021-07-15 14:15:03,008 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
67 |
+
2021-07-15 14:15:03,008 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
68 |
+
2021-07-15 14:15:14,644 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
|
69 |
+
2021-07-15 14:15:18,144 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
70 |
+
2021-07-15 14:15:18,145 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
71 |
+
2021-07-15 14:15:33,277 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
72 |
+
2021-07-15 14:15:33,278 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
73 |
+
2021-07-15 14:15:44,719 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
|
74 |
+
2021-07-15 14:15:48,414 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
75 |
+
2021-07-15 14:15:48,414 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
76 |
+
2021-07-15 14:16:03,552 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
77 |
+
2021-07-15 14:16:03,552 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
78 |
+
2021-07-15 14:16:14,795 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
|
79 |
+
2021-07-15 14:16:18,684 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
80 |
+
2021-07-15 14:16:18,685 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
81 |
+
2021-07-15 14:16:33,998 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
82 |
+
2021-07-15 14:16:33,998 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
83 |
+
2021-07-15 14:16:44,868 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
|
84 |
+
2021-07-15 14:16:49,137 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
85 |
+
2021-07-15 14:16:49,138 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
86 |
+
2021-07-15 14:17:04,268 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
87 |
+
2021-07-15 14:17:04,268 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
88 |
+
2021-07-15 14:17:14,945 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
|
89 |
+
2021-07-15 14:17:19,402 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
90 |
+
2021-07-15 14:17:19,402 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
91 |
+
2021-07-15 14:17:32,049 INFO Thread-8 :696553 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_141143-1vry5yso/files/output.log
|
92 |
+
2021-07-15 14:17:34,571 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
93 |
+
2021-07-15 14:17:34,572 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
94 |
+
2021-07-15 14:17:45,021 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
|
95 |
+
2021-07-15 14:17:49,721 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
96 |
+
2021-07-15 14:17:49,722 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
97 |
+
2021-07-15 14:18:04,866 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
98 |
+
2021-07-15 14:18:04,866 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
99 |
+
2021-07-15 14:18:15,099 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
|
100 |
+
2021-07-15 14:18:19,998 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
101 |
+
2021-07-15 14:18:19,998 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
102 |
+
2021-07-15 14:18:35,130 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
103 |
+
2021-07-15 14:18:35,130 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
104 |
+
2021-07-15 14:18:45,179 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
|
105 |
+
2021-07-15 14:18:50,263 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
106 |
+
2021-07-15 14:18:50,263 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
107 |
+
2021-07-15 14:19:05,399 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
108 |
+
2021-07-15 14:19:05,400 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
109 |
+
2021-07-15 14:19:15,259 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
|
110 |
+
2021-07-15 14:19:20,534 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
111 |
+
2021-07-15 14:19:20,534 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
112 |
+
2021-07-15 14:19:35,668 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
113 |
+
2021-07-15 14:19:35,668 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
114 |
+
2021-07-15 14:19:40,101 INFO Thread-8 :696553 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_141143-1vry5yso/files/output.log
|
115 |
+
2021-07-15 14:19:45,340 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
|
116 |
+
2021-07-15 14:19:50,825 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
117 |
+
2021-07-15 14:19:50,826 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
118 |
+
2021-07-15 14:20:05,974 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
|
119 |
+
2021-07-15 14:20:05,975 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
|
120 |
+
2021-07-15 14:20:10,445 WARNING MainThread:696553 [internal.py:wandb_internal():147] Internal process interrupt: 1
|
121 |
+
2021-07-15 14:20:10,628 WARNING MainThread:696553 [internal.py:wandb_internal():147] Internal process interrupt: 2
|
122 |
+
2021-07-15 14:20:10,628 ERROR MainThread:696553 [internal.py:wandb_internal():150] Internal process interrupted.
|
123 |
+
2021-07-15 14:20:10,822 INFO SenderThread:696553 [sender.py:finish():945] shutting down sender
|
124 |
+
2021-07-15 14:20:10,823 INFO SenderThread:696553 [dir_watcher.py:finish():282] shutting down directory watcher
|
125 |
+
2021-07-15 14:20:10,947 INFO MainThread:696553 [internal.py:handle_exit():78] Internal process exited
|
wandb/run-20210715_141143-1vry5yso/logs/debug.log
ADDED
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
2021-07-15 14:11:43,789 INFO MainThread:695296 [wandb_setup.py:_flush():69] setting env: {}
|
2 |
+
2021-07-15 14:11:43,789 INFO MainThread:695296 [wandb_setup.py:_flush():69] setting login settings: {}
|
3 |
+
2021-07-15 14:11:43,789 INFO MainThread:695296 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210715_141143-1vry5yso/logs/debug.log
|
4 |
+
2021-07-15 14:11:43,789 INFO MainThread:695296 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210715_141143-1vry5yso/logs/debug-internal.log
|
5 |
+
2021-07-15 14:11:43,790 INFO MainThread:695296 [wandb_init.py:init():370] calling init triggers
|
6 |
+
2021-07-15 14:11:43,790 INFO MainThread:695296 [wandb_init.py:init():375] wandb.init called with sweep_config: {}
|
7 |
+
config: {}
|
8 |
+
2021-07-15 14:11:43,790 INFO MainThread:695296 [wandb_init.py:init():419] starting backend
|
9 |
+
2021-07-15 14:11:43,790 INFO MainThread:695296 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
|
10 |
+
2021-07-15 14:11:43,866 INFO MainThread:695296 [backend.py:ensure_launched():135] starting backend process...
|
11 |
+
2021-07-15 14:11:43,921 INFO MainThread:695296 [backend.py:ensure_launched():139] started backend process with pid: 696553
|
12 |
+
2021-07-15 14:11:43,923 INFO MainThread:695296 [wandb_init.py:init():424] backend started and connected
|
13 |
+
2021-07-15 14:11:43,926 INFO MainThread:695296 [wandb_init.py:init():472] updated telemetry
|
14 |
+
2021-07-15 14:11:43,927 INFO MainThread:695296 [wandb_init.py:init():491] communicating current version
|
15 |
+
2021-07-15 14:11:44,548 INFO MainThread:695296 [wandb_init.py:init():496] got version response
|
16 |
+
2021-07-15 14:11:44,548 INFO MainThread:695296 [wandb_init.py:init():504] communicating run to backend with 30 second timeout
|
17 |
+
2021-07-15 14:11:44,849 INFO MainThread:695296 [wandb_init.py:init():529] starting run threads in backend
|
18 |
+
2021-07-15 14:11:46,139 INFO MainThread:695296 [wandb_run.py:_console_start():1623] atexit reg
|
19 |
+
2021-07-15 14:11:46,140 INFO MainThread:695296 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT
|
20 |
+
2021-07-15 14:11:46,141 INFO MainThread:695296 [wandb_run.py:_redirect():1502] Redirecting console.
|
21 |
+
2021-07-15 14:11:46,142 INFO MainThread:695296 [wandb_run.py:_redirect():1558] Redirects installed.
|
22 |
+
2021-07-15 14:11:46,143 INFO MainThread:695296 [wandb_init.py:init():554] run started, returning control to user process
|
23 |
+
2021-07-15 14:11:46,148 INFO MainThread:695296 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 3e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 2.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul15_14-11-35_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 50, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 10000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 2000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': './', 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''}
|
24 |
+
2021-07-15 14:11:46,150 INFO MainThread:695296 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'}
|
25 |
+
2021-07-15 14:11:46,152 INFO MainThread:695296 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 5000}
|
26 |
+
2021-07-15 14:21:53,633 INFO MainThread:695296 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 255
|
27 |
+
2021-07-15 14:21:53,634 INFO MainThread:695296 [wandb_run.py:_restore():1565] restore
|
28 |
+
2021-07-15 14:21:56,528 INFO MainThread:695296 [wandb_run.py:_restore():1565] restore
|
wandb/run-20210715_141143-1vry5yso/run-1vry5yso.wandb
ADDED
Binary file (8.12 kB). View file
|
|
wandb/run-20210715_142215-1vry5yso/files/config.yaml
ADDED
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
wandb_version: 1
|
2 |
+
|
3 |
+
_wandb:
|
4 |
+
desc: null
|
5 |
+
value:
|
6 |
+
cli_version: 0.10.33
|
7 |
+
framework: huggingface
|
8 |
+
huggingface_version: 4.9.0.dev0
|
9 |
+
is_jupyter_run: false
|
10 |
+
is_kaggle_kernel: false
|
11 |
+
python_version: 3.8.10
|
12 |
+
t:
|
13 |
+
1:
|
14 |
+
- 3
|
15 |
+
- 11
|
16 |
+
4: 3.8.10
|
17 |
+
5: 0.10.33
|
18 |
+
6: 4.9.0.dev0
|
19 |
+
8:
|
20 |
+
- 5
|
wandb/run-20210715_142215-1vry5yso/logs/debug-internal.log
ADDED
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
2021-07-15 14:22:16,240 INFO MainThread:698894 [internal.py:wandb_internal():88] W&B internal server running at pid: 698894, started at: 2021-07-15 14:22:16.240324
|
2 |
+
2021-07-15 14:22:16,242 DEBUG HandlerThread:698894 [handler.py:handle_request():124] handle_request: check_version
|
3 |
+
2021-07-15 14:22:16,242 INFO WriterThread:698894 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210715_142215-1vry5yso/run-1vry5yso.wandb
|
4 |
+
2021-07-15 14:22:16,243 DEBUG SenderThread:698894 [sender.py:send():179] send: header
|
5 |
+
2021-07-15 14:22:16,244 DEBUG SenderThread:698894 [sender.py:send_request():193] send_request: check_version
|
6 |
+
2021-07-15 14:22:16,280 DEBUG SenderThread:698894 [sender.py:send():179] send: run
|
7 |
+
2021-07-15 14:22:16,282 INFO SenderThread:698894 [sender.py:_maybe_setup_resume():413] checking resume status for wandb/hf-flax-pino-roberta/1vry5yso
|
8 |
+
2021-07-15 14:22:16,566 ERROR SenderThread:698894 [internal_api.py:execute():130] 409 response executing GraphQL.
|
9 |
+
2021-07-15 14:22:16,567 ERROR SenderThread:698894 [internal_api.py:execute():131] {"errors":[{"message":"Error 1062: Duplicate entry '370253-1vry5yso' for key 'PRIMARY'","path":["upsertBucket"]}],"data":{"upsertBucket":null}}
|
10 |
+
2021-07-15 14:22:17,890 ERROR SenderThread:698894 [internal_api.py:execute():130] 409 response executing GraphQL.
|
11 |
+
2021-07-15 14:22:17,890 ERROR SenderThread:698894 [internal_api.py:execute():131] {"errors":[{"message":"Error 1062: Duplicate entry '370253-1vry5yso' for key 'PRIMARY'","path":["upsertBucket"]}],"data":{"upsertBucket":null}}
|
12 |
+
2021-07-15 14:22:20,498 ERROR SenderThread:698894 [internal_api.py:execute():130] 409 response executing GraphQL.
|
13 |
+
2021-07-15 14:22:20,498 ERROR SenderThread:698894 [internal_api.py:execute():131] {"errors":[{"message":"Error 1062: Duplicate entry '370253-1vry5yso' for key 'PRIMARY'","path":["upsertBucket"]}],"data":{"upsertBucket":null}}
|
14 |
+
2021-07-15 14:22:20,499 ERROR SenderThread:698894 [retry.py:__call__():126] Retry attempt failed:
|
15 |
+
Traceback (most recent call last):
|
16 |
+
File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/lib/retry.py", line 102, in __call__
|
17 |
+
result = self._call_fn(*args, **kwargs)
|
18 |
+
File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/internal/internal_api.py", line 133, in execute
|
19 |
+
six.reraise(*sys.exc_info())
|
20 |
+
File "/home/dat/pino/lib/python3.8/site-packages/six.py", line 703, in reraise
|
21 |
+
raise value
|
22 |
+
File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/internal/internal_api.py", line 127, in execute
|
23 |
+
return self.client.execute(*args, **kwargs)
|
24 |
+
File "/home/dat/pino/lib/python3.8/site-packages/wandb/vendor/gql-0.2.0/gql/client.py", line 52, in execute
|
25 |
+
result = self._get_result(document, *args, **kwargs)
|
26 |
+
File "/home/dat/pino/lib/python3.8/site-packages/wandb/vendor/gql-0.2.0/gql/client.py", line 60, in _get_result
|
27 |
+
return self.transport.execute(document, *args, **kwargs)
|
28 |
+
File "/home/dat/pino/lib/python3.8/site-packages/wandb/vendor/gql-0.2.0/gql/transport/requests.py", line 39, in execute
|
29 |
+
request.raise_for_status()
|
30 |
+
File "/home/dat/pino/lib/python3.8/site-packages/requests/models.py", line 953, in raise_for_status
|
31 |
+
raise HTTPError(http_error_msg, response=self)
|
32 |
+
requests.exceptions.HTTPError: 409 Client Error: Conflict for url: https://api.wandb.ai/graphql
|
33 |
+
2021-07-15 14:22:25,627 ERROR SenderThread:698894 [internal_api.py:execute():130] 409 response executing GraphQL.
|
34 |
+
2021-07-15 14:22:25,627 ERROR SenderThread:698894 [internal_api.py:execute():131] {"errors":[{"message":"Error 1062: Duplicate entry '370253-1vry5yso' for key 'PRIMARY'","path":["upsertBucket"]}],"data":{"upsertBucket":null}}
|
35 |
+
2021-07-15 14:22:33,972 ERROR SenderThread:698894 [internal_api.py:execute():130] 409 response executing GraphQL.
|
36 |
+
2021-07-15 14:22:33,972 ERROR SenderThread:698894 [internal_api.py:execute():131] {"errors":[{"message":"Error 1062: Duplicate entry '370253-1vry5yso' for key 'PRIMARY'","path":["upsertBucket"]}],"data":{"upsertBucket":null}}
|
37 |
+
2021-07-15 14:22:46,281 DEBUG HandlerThread:698894 [handler.py:handle_request():124] handle_request: shutdown
|
38 |
+
2021-07-15 14:22:46,281 INFO HandlerThread:698894 [handler.py:finish():638] shutting down handler
|
39 |
+
2021-07-15 14:22:46,286 INFO WriterThread:698894 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210715_142215-1vry5yso/run-1vry5yso.wandb
|
40 |
+
2021-07-15 14:22:52,665 ERROR SenderThread:698894 [internal_api.py:execute():130] 409 response executing GraphQL.
|
41 |
+
2021-07-15 14:22:52,665 ERROR SenderThread:698894 [internal_api.py:execute():131] {"errors":[{"message":"Error 1062: Duplicate entry '370253-1vry5yso' for key 'PRIMARY'","path":["upsertBucket"]}],"data":{"upsertBucket":null}}
|
42 |
+
2021-07-15 14:23:25,999 ERROR SenderThread:698894 [internal_api.py:execute():130] 409 response executing GraphQL.
|
43 |
+
2021-07-15 14:23:25,999 ERROR SenderThread:698894 [internal_api.py:execute():131] {"errors":[{"message":"Error 1062: Duplicate entry '370253-1vry5yso' for key 'PRIMARY'","path":["upsertBucket"]}],"data":{"upsertBucket":null}}
|
44 |
+
2021-07-15 14:24:44,355 ERROR SenderThread:698894 [internal_api.py:execute():130] 409 response executing GraphQL.
|
45 |
+
2021-07-15 14:24:44,355 ERROR SenderThread:698894 [internal_api.py:execute():131] {"errors":[{"message":"Error 1062: Duplicate entry '370253-1vry5yso' for key 'PRIMARY'","path":["upsertBucket"]}],"data":{"upsertBucket":null}}
|
46 |
+
2021-07-15 14:27:09,800 INFO MainThread:698894 [internal.py:handle_exit():78] Internal process exited
|
wandb/run-20210715_142215-1vry5yso/logs/debug.log
ADDED
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
2021-07-15 14:22:15,517 INFO MainThread:697638 [wandb_setup.py:_flush():69] setting env: {}
|
2 |
+
2021-07-15 14:22:15,517 INFO MainThread:697638 [wandb_setup.py:_flush():69] setting login settings: {}
|
3 |
+
2021-07-15 14:22:15,517 INFO MainThread:697638 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210715_142215-1vry5yso/logs/debug.log
|
4 |
+
2021-07-15 14:22:15,517 INFO MainThread:697638 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210715_142215-1vry5yso/logs/debug-internal.log
|
5 |
+
2021-07-15 14:22:15,517 INFO MainThread:697638 [wandb_init.py:init():370] calling init triggers
|
6 |
+
2021-07-15 14:22:15,517 INFO MainThread:697638 [wandb_init.py:init():375] wandb.init called with sweep_config: {}
|
7 |
+
config: {}
|
8 |
+
2021-07-15 14:22:15,517 INFO MainThread:697638 [wandb_init.py:init():419] starting backend
|
9 |
+
2021-07-15 14:22:15,518 INFO MainThread:697638 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
|
10 |
+
2021-07-15 14:22:15,565 INFO MainThread:697638 [backend.py:ensure_launched():135] starting backend process...
|
11 |
+
2021-07-15 14:22:15,610 INFO MainThread:697638 [backend.py:ensure_launched():139] started backend process with pid: 698894
|
12 |
+
2021-07-15 14:22:15,612 INFO MainThread:697638 [wandb_init.py:init():424] backend started and connected
|
13 |
+
2021-07-15 14:22:15,615 INFO MainThread:697638 [wandb_init.py:init():472] updated telemetry
|
14 |
+
2021-07-15 14:22:15,616 INFO MainThread:697638 [wandb_init.py:init():491] communicating current version
|
15 |
+
2021-07-15 14:22:16,279 INFO MainThread:697638 [wandb_init.py:init():496] got version response
|
16 |
+
2021-07-15 14:22:16,279 INFO MainThread:697638 [wandb_init.py:init():504] communicating run to backend with 30 second timeout
|
17 |
+
2021-07-15 14:22:46,280 INFO MainThread:697638 [interface.py:_communicate_run():657] couldn't get run from backend
|
18 |
+
2021-07-15 14:22:46,280 ERROR MainThread:697638 [wandb_init.py:init():508] backend process timed out
|
19 |
+
2021-07-15 14:22:46,280 ERROR MainThread:697638 [wandb_init.py:init():517] encountered error: Error communicating with wandb process
|
20 |
+
try: wandb.init(settings=wandb.Settings(start_method='fork'))
|
21 |
+
or: wandb.init(settings=wandb.Settings(start_method='thread'))
|
22 |
+
For more info see: https://docs.wandb.ai/library/init#init-start-error
|
23 |
+
2021-07-15 14:27:09,566 WARNING MainThread:697638 [wandb_init.py:init():781] interrupted
|
24 |
+
Traceback (most recent call last):
|
25 |
+
File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/wandb_init.py", line 761, in init
|
26 |
+
run = wi.init()
|
27 |
+
File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/wandb_init.py", line 520, in init
|
28 |
+
backend.cleanup()
|
29 |
+
File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/backend/backend.py", line 168, in cleanup
|
30 |
+
self.wandb_process.join()
|
31 |
+
File "/usr/lib/python3.8/multiprocessing/process.py", line 149, in join
|
32 |
+
res = self._popen.wait(timeout)
|
33 |
+
File "/usr/lib/python3.8/multiprocessing/popen_fork.py", line 47, in wait
|
34 |
+
return self.poll(os.WNOHANG if timeout == 0.0 else 0)
|
35 |
+
File "/usr/lib/python3.8/multiprocessing/popen_fork.py", line 27, in poll
|
36 |
+
pid, sts = os.waitpid(self.pid, flag)
|
37 |
+
KeyboardInterrupt
|
wandb/run-20210715_142215-1vry5yso/run-1vry5yso.wandb
ADDED
Binary file (194 Bytes). View file
|
|
wandb/wandb-resume.json
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
{"run_id": "1vry5yso"}
|