Saving weights and logs of step 1800

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6941dc5a4d104c3446b03a4ab23a8e218ddfcc563df0533d52aafdd5b1ea91fa
+oid sha256:9974ca32d23b0ff8cd579b6068b30c9dc5ace6979b8c5c0de077bee6c04850c1
 size 891548548

run_t5.sh

@@ -26,6 +26,7 @@ mkdir -p "${MODEL_DIR}/runs"
     --logging_steps="50" \
     --save_steps="300" \
     --eval_steps="1000000" \
+    --resume_from_checkpoint="${MODEL_DIR}/ckpt-1500" \
     --push_to_hub
 
 #git add pytorch_model.bin
@@ -35,4 +36,3 @@ mkdir -p "${MODEL_DIR}/runs"
 #    --learning_rate="5e-3" \
 #    --gradient_accumulation_steps="2" \
 
-#    --resume_from_checkpoint="${MODEL_DIR}/ckpt-3300" \

run_t5_mlm_flax_custom_dataset.py

@@ -539,15 +539,21 @@ if __name__ == "__main__":
             def add_jsonlines_dir(path, filespec):
                 global data_files
                 data_files += glob.glob(f"{path}/{filespec}")
-                print(f"Number of files {len(data_files)} after adding {path}")
+                data_files = list(set(data_files))
+                print(f"Number of files {len(data_files)} after adding {path} glob {filespec}")
 
             # add_jsonlines_dir(f"{data_dir}/oscar_nl_cleaned")
             add_jsonlines_dir(f"{data_dir}/c4_cleaned", "*47*.gz")
+            add_jsonlines_dir(f"{data_dir}/c4_cleaned", "*73*.gz")
+            add_jsonlines_dir(f"{data_dir}/c4_cleaned", "*12*.gz")
+            add_jsonlines_dir(f"{data_dir}/c4_cleaned", "*29*.gz")
+            add_jsonlines_dir(f"{data_dir}/c4_cleaned", "*74*.gz")
+            add_jsonlines_dir(f"{data_dir}/c4_cleaned", "*26*.gz")
+            add_jsonlines_dir(f"{data_dir}/c4_cleaned", "*54*.gz")
             add_jsonlines_dir(f"{data_dir}/nrc_uniq_cleaned_20210223", "*.gz")
             add_jsonlines_dir(f"{data_dir}/nu_uniq_cleaned_20210225", "*.gz")
             random.Random(SEED).shuffle(data_files)
 
-            print(data_files)
             total = len(data_files)
             print(total)
             perc = 0.05
@@ -555,7 +561,7 @@ if __name__ == "__main__":
             train_size = total - val_size
             train = data_files[:train_size]
             val = data_files[train_size:]
-            print(f"Got {len(train)} training files and {perc*100} % {len(val)} validation files")
+            print(f"Got {len(train)} training files and {perc * 100} % {len(val)} validation files")
 
             assert list(set(train) & set(val)) == [], "Train overlaps with test"
 

runs/Jul10_08-38-10_t1v-n-0e7426e8-w-0/events.out.tfevents.1625906314.t1v-n-0e7426e8-w-0.25839.3.v2

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:27a53e23d145db9a5d56251a8740615ad8868024ded422ac3361448b37fca15f
-size 227319
+oid sha256:59007128de97ddf2f570d88ff0648750c666c92c091b6c87561e3fb035afb4dd
+size 259155

runs/Jul10_09-56-52_t1v-n-0e7426e8-w-0/events.out.tfevents.1625912272.t1v-n-0e7426e8-w-0.33788.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:627d89553b28ce290576ab6fc5ace8c1eb487b1230497a2a37482f94c102a957
+size 44554