Training in progress, step 198, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:33b1027b21df86a9ac1c25a185657bca1afb488a02b9101ce864a4e74d409fce
+oid sha256:92aa718963ff1a158ab63158709261c7329af2ae34ebe9805357cdb7a33e38de
 size 763470136

last-checkpoint/global_step198/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:184ca24d308598fcfdc1eff063b3844a9fe07c83450ba68204baea754acd2b6c
+size 289064656

last-checkpoint/global_step198/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f90faf99e52bcaf7883e7d0d984167b797b6a3ef426ddc5a01dcd116fa6216b6
+size 289064656

last-checkpoint/global_step198/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6222a03d2881b2d32dff03e27d53cab95de99b56844cb51944277c03b2573f1d
+size 289064656

last-checkpoint/global_step198/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:822222cd909aa8f2f0089f7216c587624099a193d544e662c584574b496ef1bb
+size 289064656

last-checkpoint/global_step198/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38e9c407ef9c94d80e6849be71df10c0f78dd80597b665cc36480ec6de1d2f3a
+size 289064656

last-checkpoint/global_step198/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36b716a34fca1e67d6f52f91eb4a50179e59d9d516991954ab69fb4a5eea15a5
+size 289064656

last-checkpoint/global_step198/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46e90af09b5072fd5c2aa766837b4c41d6c20f4942eebb5720e785ffe39e336a
+size 289064656

last-checkpoint/global_step198/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c59850584cb3af418bce4f87188bafc18552879efd66684c6a43ab0be85edf6
+size 289064656

last-checkpoint/global_step198/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05e5052d8fdfb265a4a9e86a812d17a33a0e2e85bfdb05e52437c5634475bb2a
+size 348711830

last-checkpoint/global_step198/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af29e15925b58b547d3e9448345ec03f39e0987c8eb8855f21e9cedfea0b0546
+size 348711830

last-checkpoint/global_step198/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a70597656b52052cfcb1b6823598ef703475dd92dac4d0f1b654c0315dff6c14
+size 348711830

last-checkpoint/global_step198/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b18c2b5468e85c7def27a12217ccf2c251cd8cd7d5abafe5559592bed2944108
+size 348711830

last-checkpoint/global_step198/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13ea2b7e64c74d422fb18d911fe150c9644c1417470ada198c167c2462dc9440
+size 348711830

last-checkpoint/global_step198/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3226f57d0eda0cb9f14ea96069c4660e02a76a78729fb66f1c1d15c017a929d
+size 348711830

last-checkpoint/global_step198/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6f6e07560a2a29eb4d33bd00e84f2936176d5d1c1b3a087fcffd977c882671d
+size 348711830

last-checkpoint/global_step198/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c1ffc0d5782cf1375cb74716ce8b3c27fc6cddbee62670e0bf468162a0e546e
+size 348711830

last-checkpoint/latest

@@ -1 +1 @@
-global_step132
+global_step198

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0ca402835f1af6d48f2f47ac363c7097358373e395ec83d7eb3d57ddbb0a4b2d
+oid sha256:b659790db5a549bc94a52bd0661c6c5e6c19beea5b259996f6ed9fe2149516f2
 size 15920

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a82bbd72da921737fac229854d3f27169eee4db7ddeacdba4a7199bad357bf3c
+oid sha256:4fedd424305bc76c60abafd8b0806d3107fa0fd9dcab69abdd8a175961c5d292
 size 15920

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d5972e7fa3b67599264ff7edaf7cea513fbe8d18030796597e4baae2d425cad3
+oid sha256:018b23a19c9fcba72d4cdfa2c9fc3962ed3bc3bd0e06e1ebeb979a60bbcca587
 size 15920

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b53ae0f3c148cf1921cf63943d12ae8efd4e59d00acc2c75a186e9cb04f50b9c
+oid sha256:655e1d8eba47928d19c120d020c1358f82da6b7b643dec3c9fd55e5052edd4fe
 size 15920

last-checkpoint/rng_state_4.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:84863aaa0987e7ec58181dc3d6c18d688fa5e191ef4d53a10df375a42bac5e2d
+oid sha256:0051f5950d3497fd49cd25af996fe01c32a4128ba6dc3623a168e00768ef4bd5
 size 15920

last-checkpoint/rng_state_5.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a6b191726b8c03f5e523ba50eb220f1728e82f11657f92ec30a0f367e31c0945
+oid sha256:81af3423c5f1a9239eebc7b36cb6e6db3f9862f7b90cd7560fef2590ee1d68d0
 size 15920

last-checkpoint/rng_state_6.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ae89d13f767f2c751bc315568edaf6650d050eb04a09f1b3bd9d20f069ee5007
+oid sha256:493f5e097b662c4de6f929779988d574e0855983f464da2bbac2cf6d59691a7a
 size 15920

last-checkpoint/rng_state_7.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1147b95ab7e867550d2f4e0481eddb98045dd538809f0c554423f517ebb61468
+oid sha256:47313163e11ebbda29b8bf91fd61cb4b29fc84b8ec482325f230809ff25c6426
 size 15920

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dea28a2bfa00902c551f1f93e746f32ec9126cb389e7c8deda3380b1f2fec426
+oid sha256:ced86f3a1c08cecda79ca695145fe007ebcfd4f2f8962847f6a9d9d58b4b557b
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.4049079754601227,
+  "epoch": 0.6073619631901841,
   "eval_steps": 66,
-  "global_step": 132,
+  "global_step": 198,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -955,6 +955,476 @@
       "eval_samples_per_second": 1.796,
       "eval_steps_per_second": 0.126,
       "step": 132
+    },
+    {
+      "epoch": 0.40797546012269936,
+      "grad_norm": 0.2703996506167688,
+      "learning_rate": 7.29648897747144e-05,
+      "loss": 1.9516,
+      "step": 133
+    },
+    {
+      "epoch": 0.4110429447852761,
+      "grad_norm": 0.2499546230234631,
+      "learning_rate": 7.254036429817058e-05,
+      "loss": 2.0144,
+      "step": 134
+    },
+    {
+      "epoch": 0.41411042944785276,
+      "grad_norm": 0.2755759481735348,
+      "learning_rate": 7.211399001415866e-05,
+      "loss": 1.8909,
+      "step": 135
+    },
+    {
+      "epoch": 0.4171779141104294,
+      "grad_norm": 0.25578131710544816,
+      "learning_rate": 7.168581186381824e-05,
+      "loss": 1.9747,
+      "step": 136
+    },
+    {
+      "epoch": 0.42024539877300615,
+      "grad_norm": 0.27719697668216164,
+      "learning_rate": 7.12558749784219e-05,
+      "loss": 1.9548,
+      "step": 137
+    },
+    {
+      "epoch": 0.4233128834355828,
+      "grad_norm": 0.3398789070245734,
+      "learning_rate": 7.082422467461816e-05,
+      "loss": 1.9209,
+      "step": 138
+    },
+    {
+      "epoch": 0.4263803680981595,
+      "grad_norm": 0.3891484871642631,
+      "learning_rate": 7.03909064496551e-05,
+      "loss": 1.8979,
+      "step": 139
+    },
+    {
+      "epoch": 0.4294478527607362,
+      "grad_norm": 0.28744028744457395,
+      "learning_rate": 6.995596597658468e-05,
+      "loss": 1.8568,
+      "step": 140
+    },
+    {
+      "epoch": 0.4325153374233129,
+      "grad_norm": 0.465137214109235,
+      "learning_rate": 6.951944909944877e-05,
+      "loss": 1.9201,
+      "step": 141
+    },
+    {
+      "epoch": 0.43558282208588955,
+      "grad_norm": 0.26138177619827196,
+      "learning_rate": 6.908140182844695e-05,
+      "loss": 1.9864,
+      "step": 142
+    },
+    {
+      "epoch": 0.4386503067484663,
+      "grad_norm": 0.2580799320688176,
+      "learning_rate": 6.864187033508695e-05,
+      "loss": 1.9603,
+      "step": 143
+    },
+    {
+      "epoch": 0.44171779141104295,
+      "grad_norm": 0.2342374798488655,
+      "learning_rate": 6.820090094731808e-05,
+      "loss": 1.8695,
+      "step": 144
+    },
+    {
+      "epoch": 0.4447852760736196,
+      "grad_norm": 0.31939812381318156,
+      "learning_rate": 6.775854014464799e-05,
+      "loss": 1.89,
+      "step": 145
+    },
+    {
+      "epoch": 0.44785276073619634,
+      "grad_norm": 0.3745349673551468,
+      "learning_rate": 6.731483455324374e-05,
+      "loss": 1.9072,
+      "step": 146
+    },
+    {
+      "epoch": 0.450920245398773,
+      "grad_norm": 0.2398137142916484,
+      "learning_rate": 6.686983094101712e-05,
+      "loss": 1.9224,
+      "step": 147
+    },
+    {
+      "epoch": 0.4539877300613497,
+      "grad_norm": 0.7029063348936169,
+      "learning_rate": 6.642357621269535e-05,
+      "loss": 1.9042,
+      "step": 148
+    },
+    {
+      "epoch": 0.4570552147239264,
+      "grad_norm": 0.9822378439608801,
+      "learning_rate": 6.597611740487698e-05,
+      "loss": 1.9367,
+      "step": 149
+    },
+    {
+      "epoch": 0.4601226993865031,
+      "grad_norm": 0.30640641324748263,
+      "learning_rate": 6.55275016810742e-05,
+      "loss": 1.8906,
+      "step": 150
+    },
+    {
+      "epoch": 0.46319018404907975,
+      "grad_norm": 0.28453603828616697,
+      "learning_rate": 6.507777632674165e-05,
+      "loss": 1.9607,
+      "step": 151
+    },
+    {
+      "epoch": 0.4662576687116564,
+      "grad_norm": 0.6855412180718642,
+      "learning_rate": 6.462698874429239e-05,
+      "loss": 1.8572,
+      "step": 152
+    },
+    {
+      "epoch": 0.46932515337423314,
+      "grad_norm": 0.2849104974414773,
+      "learning_rate": 6.417518644810155e-05,
+      "loss": 1.9385,
+      "step": 153
+    },
+    {
+      "epoch": 0.4723926380368098,
+      "grad_norm": 0.31769414398981494,
+      "learning_rate": 6.372241705949815e-05,
+      "loss": 1.8972,
+      "step": 154
+    },
+    {
+      "epoch": 0.4754601226993865,
+      "grad_norm": 0.6853208214886923,
+      "learning_rate": 6.326872830174567e-05,
+      "loss": 1.873,
+      "step": 155
+    },
+    {
+      "epoch": 0.4785276073619632,
+      "grad_norm": 0.3810470202905365,
+      "learning_rate": 6.281416799501188e-05,
+      "loss": 2.0,
+      "step": 156
+    },
+    {
+      "epoch": 0.4815950920245399,
+      "grad_norm": 0.3784628917790679,
+      "learning_rate": 6.235878405132842e-05,
+      "loss": 1.8814,
+      "step": 157
+    },
+    {
+      "epoch": 0.48466257668711654,
+      "grad_norm": 0.3427014353184805,
+      "learning_rate": 6.190262446954085e-05,
+      "loss": 1.9223,
+      "step": 158
+    },
+    {
+      "epoch": 0.48773006134969327,
+      "grad_norm": 0.46855229041092994,
+      "learning_rate": 6.144573733024922e-05,
+      "loss": 1.9059,
+      "step": 159
+    },
+    {
+      "epoch": 0.49079754601226994,
+      "grad_norm": 0.29232827174073656,
+      "learning_rate": 6.0988170790740416e-05,
+      "loss": 1.8491,
+      "step": 160
+    },
+    {
+      "epoch": 0.4938650306748466,
+      "grad_norm": 0.30132959369450213,
+      "learning_rate": 6.052997307991214e-05,
+      "loss": 1.9595,
+      "step": 161
+    },
+    {
+      "epoch": 0.49693251533742333,
+      "grad_norm": 0.3195413242096082,
+      "learning_rate": 6.007119249318945e-05,
+      "loss": 1.9063,
+      "step": 162
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 0.34517635749728204,
+      "learning_rate": 5.961187738743432e-05,
+      "loss": 1.9111,
+      "step": 163
+    },
+    {
+      "epoch": 0.5030674846625767,
+      "grad_norm": 0.2593428730143879,
+      "learning_rate": 5.9152076175848594e-05,
+      "loss": 1.9011,
+      "step": 164
+    },
+    {
+      "epoch": 0.5061349693251533,
+      "grad_norm": 0.31658622781595325,
+      "learning_rate": 5.86918373228712e-05,
+      "loss": 1.9918,
+      "step": 165
+    },
+    {
+      "epoch": 0.50920245398773,
+      "grad_norm": 0.6628038110211543,
+      "learning_rate": 5.8231209339069746e-05,
+      "loss": 1.9152,
+      "step": 166
+    },
+    {
+      "epoch": 0.5122699386503068,
+      "grad_norm": 0.2797312671008732,
+      "learning_rate": 5.777024077602744e-05,
+      "loss": 1.868,
+      "step": 167
+    },
+    {
+      "epoch": 0.5153374233128835,
+      "grad_norm": 0.26640093514522606,
+      "learning_rate": 5.730898022122554e-05,
+      "loss": 1.8938,
+      "step": 168
+    },
+    {
+      "epoch": 0.5184049079754601,
+      "grad_norm": 0.4054825634426873,
+      "learning_rate": 5.6847476292922155e-05,
+      "loss": 1.9428,
+      "step": 169
+    },
+    {
+      "epoch": 0.5214723926380368,
+      "grad_norm": 0.29142731230985613,
+      "learning_rate": 5.6385777635027684e-05,
+      "loss": 1.8903,
+      "step": 170
+    },
+    {
+      "epoch": 0.5245398773006135,
+      "grad_norm": 0.3511142336480421,
+      "learning_rate": 5.5923932911977575e-05,
+      "loss": 1.9386,
+      "step": 171
+    },
+    {
+      "epoch": 0.5276073619631901,
+      "grad_norm": 0.5560176165666619,
+      "learning_rate": 5.5461990803603045e-05,
+      "loss": 1.9562,
+      "step": 172
+    },
+    {
+      "epoch": 0.5306748466257669,
+      "grad_norm": 0.3171565471545065,
+      "learning_rate": 5.500000000000001e-05,
+      "loss": 1.9565,
+      "step": 173
+    },
+    {
+      "epoch": 0.5337423312883436,
+      "grad_norm": 0.29095744910567595,
+      "learning_rate": 5.4538009196396966e-05,
+      "loss": 1.9282,
+      "step": 174
+    },
+    {
+      "epoch": 0.5368098159509203,
+      "grad_norm": 0.41192796716349284,
+      "learning_rate": 5.407606708802244e-05,
+      "loss": 1.918,
+      "step": 175
+    },
+    {
+      "epoch": 0.5398773006134969,
+      "grad_norm": 0.5305521764688194,
+      "learning_rate": 5.361422236497235e-05,
+      "loss": 1.9096,
+      "step": 176
+    },
+    {
+      "epoch": 0.5429447852760736,
+      "grad_norm": 0.6434585908707302,
+      "learning_rate": 5.315252370707786e-05,
+      "loss": 1.8935,
+      "step": 177
+    },
+    {
+      "epoch": 0.5460122699386503,
+      "grad_norm": 0.2939723397914849,
+      "learning_rate": 5.2691019778774465e-05,
+      "loss": 1.9531,
+      "step": 178
+    },
+    {
+      "epoch": 0.549079754601227,
+      "grad_norm": 0.4989500512121766,
+      "learning_rate": 5.2229759223972574e-05,
+      "loss": 1.9341,
+      "step": 179
+    },
+    {
+      "epoch": 0.5521472392638037,
+      "grad_norm": 0.6024485433735285,
+      "learning_rate": 5.1768790660930265e-05,
+      "loss": 1.9001,
+      "step": 180
+    },
+    {
+      "epoch": 0.5552147239263804,
+      "grad_norm": 0.47950946229716923,
+      "learning_rate": 5.130816267712881e-05,
+      "loss": 1.9209,
+      "step": 181
+    },
+    {
+      "epoch": 0.558282208588957,
+      "grad_norm": 1.2341600337232164,
+      "learning_rate": 5.0847923824151424e-05,
+      "loss": 1.977,
+      "step": 182
+    },
+    {
+      "epoch": 0.5613496932515337,
+      "grad_norm": 0.3100804420788902,
+      "learning_rate": 5.038812261256569e-05,
+      "loss": 1.9594,
+      "step": 183
+    },
+    {
+      "epoch": 0.5644171779141104,
+      "grad_norm": 0.5390046601483737,
+      "learning_rate": 4.992880750681056e-05,
+      "loss": 1.8533,
+      "step": 184
+    },
+    {
+      "epoch": 0.5674846625766872,
+      "grad_norm": 0.43167483611230206,
+      "learning_rate": 4.9470026920087876e-05,
+      "loss": 1.8782,
+      "step": 185
+    },
+    {
+      "epoch": 0.5705521472392638,
+      "grad_norm": 0.3684508227191539,
+      "learning_rate": 4.901182920925961e-05,
+      "loss": 1.8684,
+      "step": 186
+    },
+    {
+      "epoch": 0.5736196319018405,
+      "grad_norm": 0.2936392864589777,
+      "learning_rate": 4.8554262669750794e-05,
+      "loss": 1.8586,
+      "step": 187
+    },
+    {
+      "epoch": 0.5766871165644172,
+      "grad_norm": 0.3204686860443095,
+      "learning_rate": 4.809737553045916e-05,
+      "loss": 1.8977,
+      "step": 188
+    },
+    {
+      "epoch": 0.5797546012269938,
+      "grad_norm": 0.3024045894502796,
+      "learning_rate": 4.764121594867157e-05,
+      "loss": 1.8882,
+      "step": 189
+    },
+    {
+      "epoch": 0.5828220858895705,
+      "grad_norm": 0.40522790311176354,
+      "learning_rate": 4.718583200498814e-05,
+      "loss": 1.924,
+      "step": 190
+    },
+    {
+      "epoch": 0.5858895705521472,
+      "grad_norm": 0.5053931616075322,
+      "learning_rate": 4.673127169825433e-05,
+      "loss": 1.8868,
+      "step": 191
+    },
+    {
+      "epoch": 0.588957055214724,
+      "grad_norm": 0.3211686422583536,
+      "learning_rate": 4.627758294050185e-05,
+      "loss": 1.9068,
+      "step": 192
+    },
+    {
+      "epoch": 0.5920245398773006,
+      "grad_norm": 0.24127093990601076,
+      "learning_rate": 4.582481355189846e-05,
+      "loss": 1.895,
+      "step": 193
+    },
+    {
+      "epoch": 0.5950920245398773,
+      "grad_norm": 0.4074710701692581,
+      "learning_rate": 4.537301125570763e-05,
+      "loss": 1.8969,
+      "step": 194
+    },
+    {
+      "epoch": 0.598159509202454,
+      "grad_norm": 0.25841948774460555,
+      "learning_rate": 4.492222367325837e-05,
+      "loss": 1.94,
+      "step": 195
+    },
+    {
+      "epoch": 0.6012269938650306,
+      "grad_norm": 0.2943706481314386,
+      "learning_rate": 4.447249831892583e-05,
+      "loss": 1.9482,
+      "step": 196
+    },
+    {
+      "epoch": 0.6042944785276073,
+      "grad_norm": 0.3110992148589072,
+      "learning_rate": 4.402388259512303e-05,
+      "loss": 1.9495,
+      "step": 197
+    },
+    {
+      "epoch": 0.6073619631901841,
+      "grad_norm": 0.3723312760498351,
+      "learning_rate": 4.357642378730466e-05,
+      "loss": 1.9213,
+      "step": 198
+    },
+    {
+      "epoch": 0.6073619631901841,
+      "eval_loss": 2.594010353088379,
+      "eval_runtime": 55.7716,
+      "eval_samples_per_second": 1.793,
+      "eval_steps_per_second": 0.126,
+      "step": 198
     }
   ],
   "logging_steps": 1,
@@ -974,7 +1444,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 144143397421056.0,
+  "total_flos": 216215096131584.0,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null