Update weights and scripts

Load_preprocessed_dataset.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 4,
    "id": "cf148030-7287-4c9e-ae32-8d1e1c47be30",
    "metadata": {},
    "outputs": [],
@@ -12,12 +12,30 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 7,
    "id": "5161b4ba-e8cf-43e1-b67e-503c29aa4271",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "ename": "FileNotFoundError",
+     "evalue": "[Errno 2] No such file or directory: '/home/yeb/grouped_dataset/dataset_dict.json'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
+      "\u001b[0;32m/tmp/ipykernel_574434/3668239933.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdatasets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDatasetDict\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_from_disk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"/home/yeb/grouped_dataset\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;32m~/datasets/src/datasets/dataset_dict.py\u001b[0m in \u001b[0;36mload_from_disk\u001b[0;34m(dataset_dict_path, fs, keep_in_memory)\u001b[0m\n\u001b[1;32m    727\u001b[0m                 \u001b[0;34mf\"No such file or directory: '{dataset_dict_json_path}'. Expected to load a DatasetDict object, but got a Dataset. Please use datasets.load_from_disk instead.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    728\u001b[0m             )\n\u001b[0;32m--> 729\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset_dict_json_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"r\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoding\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"utf-8\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"splits\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    730\u001b[0m             dataset_dict_split_path = (\n\u001b[1;32m    731\u001b[0m                 \u001b[0mdataset_dict_path\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"://\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"://\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mPath\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdest_dataset_dict_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_posix\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/venv/lib/python3.8/site-packages/fsspec/spec.py\u001b[0m in \u001b[0;36mopen\u001b[0;34m(self, path, mode, block_size, cache_options, **kwargs)\u001b[0m\n\u001b[1;32m    956\u001b[0m             }\n\u001b[1;32m    957\u001b[0m             return io.TextIOWrapper(\n\u001b[0;32m--> 958\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mblock_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mtext_kwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    959\u001b[0m             )\n\u001b[1;32m    960\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/venv/lib/python3.8/site-packages/fsspec/spec.py\u001b[0m in \u001b[0;36mopen\u001b[0;34m(self, path, mode, block_size, cache_options, **kwargs)\u001b[0m\n\u001b[1;32m    960\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    961\u001b[0m             \u001b[0mac\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"autocommit\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_intrans\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 962\u001b[0;31m             f = self._open(\n\u001b[0m\u001b[1;32m    963\u001b[0m                 \u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    964\u001b[0m                 \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/venv/lib/python3.8/site-packages/fsspec/implementations/local.py\u001b[0m in \u001b[0;36m_open\u001b[0;34m(self, path, mode, block_size, **kwargs)\u001b[0m\n\u001b[1;32m    142\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauto_mkdir\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m\"w\"\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    143\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmakedirs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parent\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexist_ok\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 144\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mLocalFileOpener\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    145\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    146\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mtouch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/venv/lib/python3.8/site-packages/fsspec/implementations/local.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, path, mode, autocommit, fs, compression, **kwargs)\u001b[0m\n\u001b[1;32m    233\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompression\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_compression\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcompression\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    234\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mblocksize\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDEFAULT_BUFFER_SIZE\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 235\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_open\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    236\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    237\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_open\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/venv/lib/python3.8/site-packages/fsspec/implementations/local.py\u001b[0m in \u001b[0;36m_open\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    238\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclosed\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    239\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautocommit\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m\"w\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 240\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    241\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompression\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    242\u001b[0m                     \u001b[0mcompress\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompr\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompression\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/home/yeb/grouped_dataset/dataset_dict.json'"
+     ]
+    }
+   ],
    "source": [
-    "datasets = DatasetDict.load_from_disk(\"./grouped_dataset\")"
+    "datasets = DatasetDict.load_from_disk(\"/home/yeb/grouped_dataset\")"
    ]
   },
   {

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e813a6c7760ec8638a3cf5e19b5148dad0aa0761b3c6ea82f0f74352b0308057
+oid sha256:0a2f203c6e4fb395cd1af46eddc3fdbb688e995e9d67023a95fec04d3a338d3d
 size 891548548

opt_state.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c43dcbbf24d445c2dbbc45e8fc6df64c4f79f286c21b9165e1b7d6510d498519
+oid sha256:e32d952b358ab33b2ce966fdff36cd2b253376e77f7867b0d920ea8926c0c08a
 size 1985609

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:30e5deffc7b2da9d2bba7f7b471916fd126caf80f4c0a8b1204c8f7dc62fbc3e
+oid sha256:aa8fa3e8f3ce483fb79cee84d6a4e2057ad2706010fb522c44cad201f8c0af66
 size 891650495

run_t5.sh

@@ -7,6 +7,30 @@ mkdir -p "${MODEL_DIR}/runs"
 # T5 paper lr 0.01 with batch size 128
 # We have a batch size of 8 devices * 32 = 256, so lr = 0.01/2
 
+#SEED=9200
+#
+#./run_t5_mlm_flax_custom_dataset.py \
+#    --output_dir="${MODEL_DIR}" \
+#    --model_type="t5" \
+#    --config_name="flax-community/${MODEL}" \
+#    --tokenizer_name="${MODEL_DIR}" \
+#    --seed="${SEED}" \
+#    --preprocessing_num_workers="96" \
+#    --do_train --do_eval \
+#    --adafactor \
+#    --max_seq_length="512" \
+#    --per_device_train_batch_size="32" \
+#    --per_device_eval_batch_size="32" \
+#    --dtype="bfloat16" \
+#    --learning_rate="5e-3" \
+#    --overwrite_output_dir \
+#    --num_train_epochs="3" \
+#    --logging_steps="50" \
+#    --save_steps="100" \
+#    --eval_steps="5000" \
+#    --warmup_steps="3413"
+#exit
+
 while true; do
 
   # Set the seed to random before each run, so date shuffling per epoch is different each run.
@@ -23,17 +47,17 @@ while true; do
       --do_train --do_eval \
       --adafactor \
       --max_seq_length="512" \
-      --per_device_train_batch_size="32" \
-      --per_device_eval_batch_size="32" \
-      --learning_rate="5e-3" \
+      --per_device_train_batch_size="16" \
+      --per_device_eval_batch_size="16" \
       --dtype="bfloat16" \
+      --learning_rate="1e-2" \
       --overwrite_output_dir \
       --num_train_epochs="3" \
       --logging_steps="50" \
-      --save_steps="501" \
-      --eval_steps="10000000" \
+      --save_steps="500" \
+      --eval_steps="5000" \
       --resume_from_checkpoint="${MODEL_DIR}" \
-      --warmup_steps="3413"
+      --warmup_steps="6519"
 
 #       \
 #      --push_to_hub

run_t5_mlm_flax_custom_dataset.py

@@ -645,7 +645,12 @@ if __name__ == "__main__":
 
     # Preprocessing the datasets.
     # First we tokenize all the texts.
-    if not load_grouped:
+    if load_grouped:
+        logger.info("Loading tokenized and grouped dataset")
+        tokenized_datasets = DatasetDict.load_from_disk("/home/yeb/grouped_datasets")
+        logger.info("Setting max validation examples to 500")
+        tokenized_datasets['validation'] = tokenized_datasets['validation'].select(range(500))
+    else:
         if training_args.do_train:
             column_names = datasets["train"].column_names
         else:
@@ -696,9 +701,6 @@ if __name__ == "__main__":
             num_proc=data_args.preprocessing_num_workers,
             load_from_cache_file=not data_args.overwrite_cache,
         )
-    else:
-        logger.info("Loading tokenized and grouped dataset")
-        tokenized_datasets = DatasetDict.load_from_disk("/home/yeb/grouped_datasets")
 
     # Enable tensorboard only on the master node
     has_tensorboard = is_tensorboard_available()
@@ -904,8 +906,8 @@ if __name__ == "__main__":
         for step, batch_idx in enumerate(tqdm(train_batch_idx, desc="Training...", position=1)):
             cur_step = epoch * (num_train_samples // train_batch_size) + step
             # skip to the step from which we are resuming
-#            if cur_step < resume_step:
-#                continue
+            # if cur_step < resume_step:
+            #     continue
 
             samples = [tokenized_datasets["train"][int(idx)] for idx in batch_idx]
             model_inputs = data_collator(samples)

runs/Jul11_12-53-41_t1v-n-0e7426e8-w-0/events.out.tfevents.1626008983.t1v-n-0e7426e8-w-0.161493.3.v2

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:03ddeed93b5615c1239be282f05bf781971c8a799be72c9bebc4de1d596fbd63
-size 585827

runs/Jul12_06-43-08_t1v-n-0e7426e8-w-0/events.out.tfevents.1626072193.t1v-n-0e7426e8-w-0.238699.3.v2

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9f5f6fcc83f8cf7fac87cc276fa00a02c9ce4e252c6bb69a3988452bed73f67e
-size 200238

runs/Jul13_10-43-12_t1v-n-0e7426e8-w-0/events.out.tfevents.1626172997.t1v-n-0e7426e8-w-0.622440.3.v2

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3cd61608c1fe7f600b38022829414d08eca065022eeb12ac3f3d4590930ca124
-size 96372
+oid sha256:011c4912e19d0d4a05cbfc65d26cf4419ffbb740c164af6da614e90b79e811f4
+size 370875

runs/{Jul10_08-38-10_t1v-n-0e7426e8-w-0/events.out.tfevents.1625906314.t1v-n-0e7426e8-w-0.25839.3.v2 → Jul13_13-19-07_t1v-n-0e7426e8-w-0/events.out.tfevents.1626182353.t1v-n-0e7426e8-w-0.634946.3.v2}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:59007128de97ddf2f570d88ff0648750c666c92c091b6c87561e3fb035afb4dd
-size 259155
+oid sha256:e557ecca8b2123ad7aeeae0800a959420c4d753e0700aa9b0a824e61f7a657b6
+size 1060976

runs/{Jul09_21-43-10_t1v-n-0e7426e8-w-0/events.out.tfevents.1625867209.t1v-n-0e7426e8-w-0.420316.3.v2 → Jul13_20-26-47_t1v-n-0e7426e8-w-0/events.out.tfevents.1626208012.t1v-n-0e7426e8-w-0.683072.3.v2}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ae21cbc674e3cae9cab24545f299bd4954fde9ac83daed7eca6e6b8ff17aa26e
-size 524288
+oid sha256:f51d5afdd39639a5997bd48251851300f4761d25534c61df0149123f3d7beaf3
+size 348618

runs/{Jul11_09-15-07_t1v-n-0e7426e8-w-0/events.out.tfevents.1625995853.t1v-n-0e7426e8-w-0.145718.3.v2 → Jul13_21-28-58_t1v-n-0e7426e8-w-0/events.out.tfevents.1626211744.t1v-n-0e7426e8-w-0.688568.3.v2}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f1aed2c3d6af21a006e34d03f0d6071dd41995d510e1cee680aad026186ca98f
-size 303949
+oid sha256:5874afd27d6f8234168e30694876b6bda7c58a0cdeb56e640a8ee61d6c3f3bbb
+size 2975368

runs/{Jul10_07-45-49_t1v-n-0e7426e8-w-0/events.out.tfevents.1625903173.t1v-n-0e7426e8-w-0.20563.3.v2 → Jul14_05-04-35_t1v-n-0e7426e8-w-0/events.out.tfevents.1626239081.t1v-n-0e7426e8-w-0.720177.3.v2}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:07af3cc1b9177077621a9f9cd2b3d36aa028237973caf686c9e324e89cc25de9
-size 147729
+oid sha256:c75c45dc32377c52f7a7b2df50ceded99c6098035185d14135132e0626683835
+size 51858

runs/Jul14_05-21-55_t1v-n-0e7426e8-w-0/events.out.tfevents.1626240121.t1v-n-0e7426e8-w-0.722772.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fec605fd70b4cc49e1a473d3cd871b834a2f711519812618dded0d6f397c4daf
+size 1335479

streaming_dataset_filter_test.py

@@ -4,26 +4,90 @@ from datasets import load_dataset
 
 dataset_v0 = load_dataset('oscar', "unshuffled_deduplicated_nl", split='train', streaming=True)
 
+# data_dir = "/home/yeb"
+data_dir = "/home/yeb/Developer/data"
+data_files = []
+
+def train_val_files():
+    import glob
+    import random
+    SEED = 12345
+
+    def add_jsonlines_dir(path, filespec):
+        global data_files
+        data_files += glob.glob(f"{path}/{filespec}")
+        data_files = list(set(data_files))
+        print(f"Number of files {len(data_files)} after adding {path} glob {filespec}")
+
+    # add_jsonlines_dir(f"{data_dir}/oscar_nl_cleaned")
+    add_jsonlines_dir(f"{data_dir}/c4_cleaned2", "*73*.gz")
+#     add_jsonlines_dir(f"{data_dir}/c4_cleaned", "*47*.gz")
+#     add_jsonlines_dir(f"{data_dir}/c4_cleaned", "*12*.gz")
+#     add_jsonlines_dir(f"{data_dir}/c4_cleaned", "*29*.gz")
+#     add_jsonlines_dir(f"{data_dir}/c4_cleaned", "*74*.gz")
+#     add_jsonlines_dir(f"{data_dir}/c4_cleaned", "*26*.gz")
+#     add_jsonlines_dir(f"{data_dir}/c4_cleaned", "*54*.gz")
+#     add_jsonlines_dir(f"{data_dir}/c4_cleaned", "*68*.gz")
+#     add_jsonlines_dir(f"{data_dir}/c4_cleaned", "*57*.gz")
+    # add_jsonlines_dir(f"{data_dir}/c4_cleaned", "*46*.gz")
+    # add_jsonlines_dir(f"{data_dir}/c4_cleaned", "*35*.gz")
+    # add_jsonlines_dir(f"{data_dir}/c4_cleaned", "*13*.gz")
+    # add_jsonlines_dir(f"{data_dir}/c4_cleaned", "*41*.gz")
+    # add_jsonlines_dir(f"{data_dir}/c4_cleaned", "*52*.gz")
+    # add_jsonlines_dir(f"{data_dir}/c4_cleaned", "*63*.gz")
+    # add_jsonlines_dir(f"{data_dir}/c4_cleaned", "*85*.gz")
+    # add_jsonlines_dir(f"{data_dir}/c4_cleaned", "*81*.gz")
+    # add_jsonlines_dir(f"{data_dir}/c4_cleaned", "*96*.gz")
+    # add_jsonlines_dir(f"{data_dir}/nrc_uniq_cleaned_20210223", "*.gz")
+#     add_jsonlines_dir(f"{data_dir}/nu_uniq_cleaned_20210225", "*.gz")
+    random.Random(SEED).shuffle(data_files)
+
+    total = len(data_files)
+    print(total)
+    perc = 0.05
+    val_size = int(perc * total)
+    train_size = total - val_size
+    train = data_files[:train_size]
+    val = data_files[train_size:]
+    print(f"Got {len(train)} training files and {perc * 100} % {len(val)} validation files")
+
+    assert list(set(train) & set(val)) == [], "Train overlaps with test"
+
+    return train, val
+
+train, val = train_val_files()
+dataset_v0 = load_dataset('json', data_files={'train': train, 'validation': val})
+
+
+dataset_v0 = load_dataset('oscar', "unshuffled_deduplicated_nl")
 
 def f(obj):
     obj["text"] = clean_text(obj["text"])
     return obj
 
 
-dataset_v1 = dataset_v0.map(f)
-dataset_v2 = dataset_v1.filter(lambda obj: obj['text'] is not None)
+dataset_v1 = dataset_v0.map(
+    f,
+    batched=False,
+    num_proc=10,
+)
 
-it = iter(dataset_v0)
-print(next(it))
-print(next(it))
-print(next(it))
+datasets = dataset_v1.filter(
+    lambda obj: obj['text'] is not None,
+    num_proc=10,
+)
 
-it = iter(dataset_v1)
+it = iter(dataset_v0['train'])
 print(next(it))
 print(next(it))
 print(next(it))
 
-it = iter(dataset_v2)
+it = iter(dataset_v1['train'])
 print(next(it))
 print(next(it))
 print(next(it))
+
+# it = iter(dataset_v2)
+# print(next(it))
+# print(next(it))
+# print(next(it))

training_state.json

@@ -1 +1 @@
-{"step": 502}
+{"step": 30204}