Saving at step 2840k, loss 1.108, acc 0.747

eval_results.json

@@ -0,0 +1,4 @@
+{
+    "eval_accuracy": 0.7469202280044556,
+    "eval_loss": 1.1080663204193115
+}

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:be580d25b36fead261bee39bb04610e55d681c9880ea51883d2747d24f950c28
+oid sha256:b38f40c38ee1b037c8c509dd349a3c8d3c16870fa0571f9d947c7cd1680ebd9d
 size 990170015

info.txt

@@ -0,0 +1,9 @@
+INFO:__main__:  Optimizer = adafactor
+INFO:__main__:  Learning rate (peak) = 0.005
+INFO:__main__:  Num examples = 31519126
+INFO:__main__:  Num tokenized group examples 36347268
+INFO:__main__:  Num Epochs = 10
+INFO:__main__:  Instantaneous batch size per device = 16
+INFO:__main__:  Total train batch size (w. parallel & grad accum) = 128
+INFO:__main__:  Steps per epoch = 283963
+INFO:__main__:  Total optimization steps = 2839630

opt_state.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:81e11965da1594b414d17cf95b0dc6e0e62f041352c61969a6a8a8141351df5e
+oid sha256:5871414180a92600a235367a2d7dd7c6a9ca06ac3b2311a6d87203383992474c
 size 2184331

run.sh

@@ -0,0 +1,38 @@
+export HF_PROJECT="t5-v1_1-base-dutch-english-cased"
+export DATASET="yhavinga/mc4_nl_cleaned" # Name of the dataset in the Huggingface Hub
+export DATASET_CONFIG="small_en_nl" # Config of the dataset in the Huggingface Hub
+export DATASET_SPLIT="train" # Split to use for training tokenizer and model
+export CONFIG_NAME="yhavinga/t5-v1.1-base-dutch-cased"
+export TOKENIZER_NAME="yhavinga/net5-v1.1-base-cased-500"
+export MODEL_PATH="${HOME}/data/${HF_PROJECT}" # Path to the model
+
+
+python3 ../train/run_t5_mlm_flax_pmap.py \
+    --output_dir="${MODEL_PATH}" \
+    --model_type="t5" \
+    --config_name="${CONFIG_NAME}" \
+    --tokenizer_name="${TOKENIZER_NAME}" \
+    --auth_token="$(cat ~/.huggingface/token)" \
+    --preprocessing_num_workers="96" \
+    --do_train --do_eval \
+    --dataset_name="${DATASET}" \
+    --dataset_config_name="${DATASET_CONFIG}" \
+    --max_seq_length="512" \
+    --per_device_train_batch_size="16" \
+    --per_device_eval_batch_size="16" \
+    --optim="adafactor" \
+    --learning_rate="0.005" \
+    --lr_decay="exponential" \
+    --lr_transition_steps="300000" \
+    --lr_decay_rate="0.7" \
+    --lr_staircase="false" \
+    --overwrite_output_dir \
+    --num_train_epochs="10" \
+    --logging_steps="200" \
+    --save_steps="10000" \
+    --eval_steps="1250" \
+    --warmup_steps="10000" \
+    --validation_split_count="15000" \
+    --wandb_project="t5-v1_1-dutch-english" \
+    --wandb_job_type="pmap" \
+    --resume_from_checkpoint="${MODEL_PATH}"

training_state.json

@@ -1 +1 @@
-{"step": 2119986}
+{"step": 2829975}