Saving weights and logs of step 90000

checkpoint_90000

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c62d4bdd9a95f5039224ba4d25a6b9d4c04488dfbd282b54718cf5562822965a
+size 1530270447

events.out.tfevents.1626318666.t1v-n-f5c06ea1-w-0.655476.3.v2

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c96fa1e6828a98c51c01d533b269b18d5b997aab8e26c595124e60a983f3a911
-size 8991258
+oid sha256:87d384252cfc2c9551426c799e7901b66a0c7e2351dc2b43d1d2634656ab26a9
+size 13503400

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5938f5a18f14894542ca37eeda9f7d5780b628c6b67cf598633785806d1ead78
+oid sha256:e9dd04b40d879736851e06d01940a5427c1086eabab0e17bdc29ee5016e1f264
 size 510090043

run.sh

@@ -15,17 +15,17 @@ python ./run_mlm_flax_no_accum.py \
     --adam_beta1="0.9" \
     --adam_beta2="0.98" \
     --logging_steps="50" \
-    --eval_steps="20000" \
-    --num_train_epochs="5" \
+    --eval_steps="5000" \
+    --num_train_epochs="2"\
     --preprocessing_num_workers="96" \
-    --save_steps="30000" \
+    --save_steps="10000" \
     --learning_rate="3e-5" \
     --per_device_train_batch_size="1" \
     --per_device_eval_batch_size="1" \
     --save_total_limit="5"\
-    --max_eval_samples="500"\
+    --max_eval_samples="5000"\
+    --resume_from_checkpoint="./"\
     #--gradient_accumulation_steps="4"\
-    #--resume_from_checkpoint="./"\
     #--adafactor \
     #--dtype="bfloat16" \
     

wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json

@@ -1 +1 @@
-{"training_step": 60000, "learning_rate": 2.914472861448303e-05, "train_loss": 3.118450880050659, "_runtime": 24758, "_timestamp": 1626343425, "_step": 1202, "eval_step": 60000, "eval_accuracy": 0.4923423230648041, "eval_loss": 2.8991198539733887}
+{"training_step": 90000, "learning_rate": 2.863156441890169e-05, "train_loss": 2.512800693511963, "_runtime": 36892, "_timestamp": 1626355559, "_step": 1803, "eval_step": 80000, "eval_accuracy": 0.533167839050293, "eval_loss": 2.5656967163085938}