training on large slovak corpus

config.json

@@ -21,5 +21,9 @@
   "pad_token_id": 0,
   "relative_attention_num_buckets": 32,
   "tie_word_embeddings": false,
-  "vocab_size": 64100
+  "vocab_size": 120100,
+  "sp_model_kwargs":{
+      "enable_sampling": true,
+      "nbest_size": 4
+  }
 }

hydra/config.yaml

@@ -6,36 +6,37 @@ predict_only: false
 seed: 2137
 model:
   klass: hf_t5
-  name: /home/jovyan/bert-train/nanot5/base_slovak_model/
+  name: /home/jovyan/bert-train/nanot5/templates/base_sklarge_120k
   overwrite:
     dropout_rate: 0.0
   add_config:
-    is_bf16: false
+    is_bf16: true
   checkpoint_path: ''
   random_init: true
   compile: false
 data:
+  train_path: /home/jovyan/data/sklarge-shards
   input_length: 512
   mlm_probability: 0.15
   mean_noise_span_length: 3.0
-  num_workers: 2
+  num_workers: 8
 optim:
-  name: adamwscale
+  name: adafactor
   base_lr: 0.02
   batch_size: 128
-  total_steps: 65536
+  total_steps: 120000
   epochs: -1
   warmup_steps: 10000
-  lr_scheduler: cosine
+  lr_scheduler: legacy
   weight_decay: 0.0
   grad_clip: 1.0
-  grad_acc: 4
+  grad_acc: 8
   final_cosine: 1.0e-05
 eval:
   every_steps: 5000
   steps: 500
 checkpoint:
-  every_steps: 20000
+  every_steps: 10000
 logging:
   neptune: false
   neptune_creds:

hydra/hydra.yaml

@@ -112,17 +112,21 @@ hydra:
     hydra:
     - hydra.mode=RUN
     task:
-    - optim.name=adamwscale
-    - optim.lr_scheduler=cosine
-    - model.name=/home/jovyan/bert-train/nanot5/base_slovak_model/
-    - optim.grad_acc=4
+    - optim.name=adafactor
+    - optim.lr_scheduler=legacy
+    - model.name=/home/jovyan/bert-train/nanot5/templates/base_sklarge_120k
+    - data.train_path=/home/jovyan/data/sklarge-shards
+    - optim.grad_acc=8
     - model.klass=hf_t5
     - eval.every_steps=5000
-    - checkpoint.every_steps=20000
+    - optim.total_steps=120000
+    - model.add_config.is_bf16=True
+    - checkpoint.every_steps=10000
+    - model.compile=False
   job:
     name: main
     chdir: true
-    override_dirname: checkpoint.every_steps=20000,eval.every_steps=5000,model.klass=hf_t5,model.name=/home/jovyan/bert-train/nanot5/base_slovak_model/,optim.grad_acc=4,optim.lr_scheduler=cosine,optim.name=adamwscale
+    override_dirname: checkpoint.every_steps=10000,data.train_path=/home/jovyan/data/sklarge-shards,eval.every_steps=5000,model.add_config.is_bf16=True,model.compile=False,model.klass=hf_t5,model.name=/home/jovyan/bert-train/nanot5/templates/base_sklarge_120k,optim.grad_acc=8,optim.lr_scheduler=legacy,optim.name=adafactor,optim.total_steps=120000
     id: ???
     num: ???
     config_name: default
@@ -147,7 +151,7 @@ hydra:
     - path: ''
       schema: structured
       provider: schema
-    output_dir: /home/jovyan/nanoT5/logs/2024-01-02/07-29-30-
+    output_dir: /home/jovyan/nanoT5/logs/2024-07-29/11-48-34-
     choices:
       local_env: default
       task: pt

hydra/overrides.yaml

@@ -1,7 +1,11 @@
-- optim.name=adamwscale
-- optim.lr_scheduler=cosine
-- model.name=/home/jovyan/bert-train/nanot5/base_slovak_model/
-- optim.grad_acc=4
+- optim.name=adafactor
+- optim.lr_scheduler=legacy
+- model.name=/home/jovyan/bert-train/nanot5/templates/base_sklarge_120k
+- data.train_path=/home/jovyan/data/sklarge-shards
+- optim.grad_acc=8
 - model.klass=hf_t5
 - eval.every_steps=5000
-- checkpoint.every_steps=20000
+- optim.total_steps=120000
+- model.add_config.is_bf16=True
+- checkpoint.every_steps=10000
+- model.compile=False

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8adcaa3befd91b9080f22d00466debbce77b0989eb3b038269f83aad1ee5e934
-size 1186781032
+oid sha256:d083edc4ffae1fc5025aeec7b6417c459a1654010f0bd9fee0ae6c57ab97332f
+size 1530845040

optimizer.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5d9f3304271a05b8894b45869b7bae03ae655aa7615410d7ee722ab02064ceea
-size 2373662661
+oid sha256:b2d8ba594f622898a38b758a0bfc7a6bf1e79acde08d843d03b771a5b220106b
+size 3075653

random_states_0.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a429eefea0f594d70075b0d57721b18d7e6e4d12263218ee30780a6e613c04cd
+oid sha256:032d5c0c40c80570faf14e7a3a2ba39d76b91afecb36586b35a861be0d83556d
 size 14663

scheduler.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6b51bb232eab4bf0dd13cf507f602d257c3596e6828466eb03664de335c46223
+oid sha256:c26cfd01bde4900b7b03e0e9b7ec7d389da736cdcf1ccb3d46a64498895077f0
 size 819

spiece.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:353edf8f042581ee554bb883dd4a19e0888d778553e3fd5ca9c97bb76434406b
-size 1408434
+oid sha256:9944e5920f922d26793ec2b15ae90f576584035eecde1e9eee0923bc8c3fc328
+size 2575539

train-model.sh

@@ -0,0 +1 @@
+python -m nanoT5.main   optim.name=adafactor   optim.lr_scheduler=legacy model.name=/home/jovyan/bert-train/nanot5/templates/base_sklarge_120k data.train_path=/home/jovyan/data/sklarge-shards optim.grad_acc=8  model.klass=hf_t5 eval.every_steps=5000 optim.total_steps=120000 model.add_config.is_bf16=True checkpoint.every_steps=10000 model.compile=False