End of training

README.md

@@ -18,11 +18,11 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [riotu-lab/ArabianGPT-01B](https://huggingface.co/riotu-lab/ArabianGPT-01B) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.4497
-- Bleu: 0.4202
-- Rouge1: 0.6498
-- Rouge2: 0.3905
-- Rougel: 0.6488
+- Loss: 0.4179
+- Bleu: 0.4093
+- Rouge1: 0.6309
+- Rouge2: 0.3587
+- Rougel: 0.6298
 
 ## Model description
 

all_results.json

@@ -1,19 +1,19 @@
 {
-    "epoch": 14.0,
-    "eval_bleu": 0.23736106874390867,
-    "eval_loss": 0.4183219075202942,
-    "eval_rouge1": 0.6213217050496613,
-    "eval_rouge2": 0.3629928034660268,
-    "eval_rougeL": 0.6194945230812778,
-    "eval_runtime": 6.133,
-    "eval_samples": 1675,
-    "eval_samples_per_second": 273.111,
-    "eval_steps_per_second": 34.241,
-    "perplexity": 1.5194097050971664,
-    "total_flos": 6127298150400000.0,
-    "train_loss": 0.03383859399638808,
-    "train_runtime": 1080.9024,
-    "train_samples": 6700,
-    "train_samples_per_second": 123.97,
-    "train_steps_per_second": 15.506
+    "epoch": 9.0,
+    "eval_bleu": 0.4093436818765597,
+    "eval_loss": 0.41785159707069397,
+    "eval_rouge1": 0.6308594985430711,
+    "eval_rouge2": 0.35871585587023913,
+    "eval_rougeL": 0.6297639876498756,
+    "eval_runtime": 5.4884,
+    "eval_samples": 1672,
+    "eval_samples_per_second": 304.643,
+    "eval_steps_per_second": 38.08,
+    "perplexity": 1.5186952788498689,
+    "total_flos": 3930158776320000.0,
+    "train_loss": 0.40989596332913064,
+    "train_runtime": 1923.2663,
+    "train_samples": 6685,
+    "train_samples_per_second": 69.517,
+    "train_steps_per_second": 8.694
 }

egy_training_log.txt

@@ -162,3 +162,5 @@ INFO:root:Epoch 8.0: Train Loss = 0.1492, Eval Loss = 0.4348294138908386
 INFO:absl:Using default tokenizer.
 INFO:root:Epoch 9.0: Train Loss = 0.1293, Eval Loss = 0.44312888383865356
 INFO:absl:Using default tokenizer.
+INFO:__main__:*** Evaluate ***
+INFO:absl:Using default tokenizer.

eval_results.json

@@ -1,13 +1,13 @@
 {
-    "epoch": 14.0,
-    "eval_bleu": 0.23736106874390867,
-    "eval_loss": 0.4183219075202942,
-    "eval_rouge1": 0.6213217050496613,
-    "eval_rouge2": 0.3629928034660268,
-    "eval_rougeL": 0.6194945230812778,
-    "eval_runtime": 6.133,
-    "eval_samples": 1675,
-    "eval_samples_per_second": 273.111,
-    "eval_steps_per_second": 34.241,
-    "perplexity": 1.5194097050971664
+    "epoch": 9.0,
+    "eval_bleu": 0.4093436818765597,
+    "eval_loss": 0.41785159707069397,
+    "eval_rouge1": 0.6308594985430711,
+    "eval_rouge2": 0.35871585587023913,
+    "eval_rougeL": 0.6297639876498756,
+    "eval_runtime": 5.4884,
+    "eval_samples": 1672,
+    "eval_samples_per_second": 304.643,
+    "eval_steps_per_second": 38.08,
+    "perplexity": 1.5186952788498689
 }

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 14.0,
-    "total_flos": 6127298150400000.0,
-    "train_loss": 0.03383859399638808,
-    "train_runtime": 1080.9024,
-    "train_samples": 6700,
-    "train_samples_per_second": 123.97,
-    "train_steps_per_second": 15.506
+    "epoch": 9.0,
+    "total_flos": 3930158776320000.0,
+    "train_loss": 0.40989596332913064,
+    "train_runtime": 1923.2663,
+    "train_samples": 6685,
+    "train_samples_per_second": 69.517,
+    "train_steps_per_second": 8.694
 }

trainer_state.json

@@ -1,291 +1,196 @@
 {
-  "best_metric": 0.4183219075202942,
-  "best_model_checkpoint": "/home/iais_marenpielka/Bouthaina/res_nw_gulf/checkpoint-3352",
-  "epoch": 14.0,
+  "best_metric": 0.41785159707069397,
+  "best_model_checkpoint": "/home/iais_marenpielka/Bouthaina/res_nw_gulf/checkpoint-3344",
+  "epoch": 9.0,
   "eval_steps": 500,
-  "global_step": 11732,
+  "global_step": 7524,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 1.0,
-      "grad_norm": 1.1270569562911987,
-      "learning_rate": 4.8960639606396064e-05,
-      "loss": 1.9575,
-      "step": 838
+      "grad_norm": 1.0378659963607788,
+      "learning_rate": 4.896424167694204e-05,
+      "loss": 1.8893,
+      "step": 836
     },
     {
       "epoch": 1.0,
-      "eval_bleu": 0.17899151773967856,
-      "eval_loss": 0.4805418848991394,
-      "eval_rouge1": 0.5300700267573193,
-      "eval_rouge2": 0.25846419107204655,
-      "eval_rougeL": 0.5287371290578213,
-      "eval_runtime": 26.4765,
-      "eval_samples_per_second": 63.264,
-      "eval_steps_per_second": 7.932,
-      "step": 838
+      "eval_bleu": 0.3817836670274851,
+      "eval_loss": 0.4814806580543518,
+      "eval_rouge1": 0.53445667160678,
+      "eval_rouge2": 0.2556577725762693,
+      "eval_rougeL": 0.5334625759517182,
+      "eval_runtime": 13.5021,
+      "eval_samples_per_second": 123.833,
+      "eval_steps_per_second": 15.479,
+      "step": 836
     },
     {
       "epoch": 2.0,
-      "grad_norm": 0.8918192982673645,
-      "learning_rate": 4.6383763837638376e-05,
-      "loss": 0.422,
-      "step": 1676
+      "grad_norm": 1.1964277029037476,
+      "learning_rate": 4.638717632552405e-05,
+      "loss": 0.4211,
+      "step": 1672
     },
     {
       "epoch": 2.0,
-      "eval_bleu": 0.2111987576275603,
-      "eval_loss": 0.4326799809932709,
-      "eval_rouge1": 0.5789346222407353,
-      "eval_rouge2": 0.3115433580134526,
-      "eval_rougeL": 0.5777838651600141,
-      "eval_runtime": 36.5489,
-      "eval_samples_per_second": 45.829,
-      "eval_steps_per_second": 5.746,
-      "step": 1676
+      "eval_bleu": 0.39468460167151326,
+      "eval_loss": 0.4300891160964966,
+      "eval_rouge1": 0.5886104983083813,
+      "eval_rouge2": 0.30839577052783973,
+      "eval_rougeL": 0.5877200747673375,
+      "eval_runtime": 5.8159,
+      "eval_samples_per_second": 287.485,
+      "eval_steps_per_second": 35.936,
+      "step": 1672
     },
     {
       "epoch": 3.0,
-      "grad_norm": 1.1533405780792236,
-      "learning_rate": 4.380688806888069e-05,
-      "loss": 0.331,
-      "step": 2514
+      "grad_norm": 1.0570303201675415,
+      "learning_rate": 4.3810110974106046e-05,
+      "loss": 0.3307,
+      "step": 2508
     },
     {
       "epoch": 3.0,
-      "eval_bleu": 0.22887638181669745,
-      "eval_loss": 0.41845184564590454,
-      "eval_rouge1": 0.6066983915980586,
-      "eval_rouge2": 0.34626660980048474,
-      "eval_rougeL": 0.6055721645591957,
-      "eval_runtime": 25.3383,
-      "eval_samples_per_second": 66.106,
-      "eval_steps_per_second": 8.288,
-      "step": 2514
+      "eval_bleu": 0.40983966101019575,
+      "eval_loss": 0.4181276261806488,
+      "eval_rouge1": 0.6130185046348249,
+      "eval_rouge2": 0.33816617915935476,
+      "eval_rougeL": 0.6119109322410228,
+      "eval_runtime": 18.13,
+      "eval_samples_per_second": 92.223,
+      "eval_steps_per_second": 11.528,
+      "step": 2508
     },
     {
       "epoch": 4.0,
-      "grad_norm": 1.3615330457687378,
-      "learning_rate": 4.123001230012301e-05,
-      "loss": 0.2638,
-      "step": 3352
+      "grad_norm": 1.0588442087173462,
+      "learning_rate": 4.1233045622688044e-05,
+      "loss": 0.2641,
+      "step": 3344
     },
     {
       "epoch": 4.0,
-      "eval_bleu": 0.23736106874390867,
-      "eval_loss": 0.4183219075202942,
-      "eval_rouge1": 0.6213217050496613,
-      "eval_rouge2": 0.3629928034660268,
-      "eval_rougeL": 0.6194945230812778,
-      "eval_runtime": 16.1799,
-      "eval_samples_per_second": 103.524,
-      "eval_steps_per_second": 12.979,
-      "step": 3352
+      "eval_bleu": 0.4093436818765597,
+      "eval_loss": 0.41785159707069397,
+      "eval_rouge1": 0.6308594985430711,
+      "eval_rouge2": 0.35871585587023913,
+      "eval_rougeL": 0.6297639876498756,
+      "eval_runtime": 5.6414,
+      "eval_samples_per_second": 296.382,
+      "eval_steps_per_second": 37.048,
+      "step": 3344
     },
     {
       "epoch": 5.0,
-      "grad_norm": 1.29606294631958,
-      "learning_rate": 3.865313653136531e-05,
-      "loss": 0.2131,
-      "step": 4190
+      "grad_norm": 1.3207675218582153,
+      "learning_rate": 3.8655980271270036e-05,
+      "loss": 0.2135,
+      "step": 4180
     },
     {
       "epoch": 5.0,
-      "eval_bleu": 0.24225546513762886,
-      "eval_loss": 0.42484816908836365,
-      "eval_rouge1": 0.6280394265150125,
-      "eval_rouge2": 0.3718667234148674,
-      "eval_rougeL": 0.6264526155291095,
-      "eval_runtime": 25.4283,
-      "eval_samples_per_second": 65.872,
-      "eval_steps_per_second": 8.259,
-      "step": 4190
+      "eval_bleu": 0.41246868412584214,
+      "eval_loss": 0.4242132008075714,
+      "eval_rouge1": 0.6373073919820111,
+      "eval_rouge2": 0.36914621811033116,
+      "eval_rougeL": 0.6359208794207342,
+      "eval_runtime": 5.3736,
+      "eval_samples_per_second": 311.149,
+      "eval_steps_per_second": 38.894,
+      "step": 4180
     },
     {
       "epoch": 6.0,
-      "grad_norm": 1.9162726402282715,
-      "learning_rate": 3.6076260762607624e-05,
-      "loss": 0.1756,
-      "step": 5028
+      "grad_norm": 1.1845903396606445,
+      "learning_rate": 3.6078914919852034e-05,
+      "loss": 0.1765,
+      "step": 5016
     },
     {
       "epoch": 6.0,
-      "eval_bleu": 0.25110200021461276,
-      "eval_loss": 0.4322951138019562,
-      "eval_rouge1": 0.6329099900871186,
-      "eval_rouge2": 0.3818443278310246,
-      "eval_rougeL": 0.6311516008372622,
-      "eval_runtime": 13.3457,
-      "eval_samples_per_second": 125.508,
-      "eval_steps_per_second": 15.735,
-      "step": 5028
+      "eval_bleu": 0.41560183658228855,
+      "eval_loss": 0.43002957105636597,
+      "eval_rouge1": 0.6465581796614536,
+      "eval_rouge2": 0.382979021316462,
+      "eval_rougeL": 0.6453232583485639,
+      "eval_runtime": 5.4874,
+      "eval_samples_per_second": 304.7,
+      "eval_steps_per_second": 38.088,
+      "step": 5016
     },
     {
       "epoch": 7.0,
-      "grad_norm": 1.2213388681411743,
-      "learning_rate": 3.349938499384994e-05,
-      "loss": 0.148,
-      "step": 5866
+      "grad_norm": 0.9745954871177673,
+      "learning_rate": 3.350184956843403e-05,
+      "loss": 0.1492,
+      "step": 5852
     },
     {
       "epoch": 7.0,
-      "eval_bleu": 0.24584223218153298,
-      "eval_loss": 0.4369480013847351,
-      "eval_rouge1": 0.6346640667719812,
-      "eval_rouge2": 0.3890963668887653,
-      "eval_rougeL": 0.6326978702709232,
-      "eval_runtime": 6.3313,
-      "eval_samples_per_second": 264.559,
-      "eval_steps_per_second": 33.169,
-      "step": 5866
+      "eval_bleu": 0.41879193694803385,
+      "eval_loss": 0.4348294138908386,
+      "eval_rouge1": 0.6460454718776507,
+      "eval_rouge2": 0.3852436451416871,
+      "eval_rougeL": 0.6448992585011752,
+      "eval_runtime": 12.7816,
+      "eval_samples_per_second": 130.813,
+      "eval_steps_per_second": 16.352,
+      "step": 5852
     },
     {
       "epoch": 8.0,
-      "grad_norm": 1.774276852607727,
-      "learning_rate": 3.092250922509225e-05,
-      "loss": 0.1285,
-      "step": 6704
+      "grad_norm": 0.8118007779121399,
+      "learning_rate": 3.092478421701603e-05,
+      "loss": 0.1293,
+      "step": 6688
     },
     {
       "epoch": 8.0,
-      "eval_bleu": 0.2499016538097514,
-      "eval_loss": 0.4447513520717621,
-      "eval_rouge1": 0.6380175687280993,
-      "eval_rouge2": 0.38845767776215345,
-      "eval_rougeL": 0.6368182514454324,
-      "eval_runtime": 5.4681,
-      "eval_samples_per_second": 306.322,
-      "eval_steps_per_second": 38.405,
-      "step": 6704
+      "eval_bleu": 0.4190463110305805,
+      "eval_loss": 0.44312888383865356,
+      "eval_rouge1": 0.6492735112653347,
+      "eval_rouge2": 0.389473573727392,
+      "eval_rougeL": 0.6482287927411368,
+      "eval_runtime": 5.4157,
+      "eval_samples_per_second": 308.729,
+      "eval_steps_per_second": 38.591,
+      "step": 6688
     },
     {
       "epoch": 9.0,
-      "grad_norm": 0.779574990272522,
-      "learning_rate": 2.8345633456334564e-05,
-      "loss": 0.1152,
-      "step": 7542
+      "grad_norm": 0.9986652135848999,
+      "learning_rate": 2.8347718865598028e-05,
+      "loss": 0.1155,
+      "step": 7524
     },
     {
       "epoch": 9.0,
-      "eval_bleu": 0.2538776211394201,
-      "eval_loss": 0.45251065492630005,
-      "eval_rouge1": 0.6407467681145795,
-      "eval_rouge2": 0.3976657405838769,
-      "eval_rougeL": 0.6393461853572592,
-      "eval_runtime": 6.3546,
-      "eval_samples_per_second": 263.59,
-      "eval_steps_per_second": 33.047,
-      "step": 7542
+      "eval_bleu": 0.4202341801445125,
+      "eval_loss": 0.449709415435791,
+      "eval_rouge1": 0.6498383797626817,
+      "eval_rouge2": 0.39052732019696323,
+      "eval_rougeL": 0.6488280555236853,
+      "eval_runtime": 5.4735,
+      "eval_samples_per_second": 305.473,
+      "eval_steps_per_second": 38.184,
+      "step": 7524
     },
     {
-      "epoch": 10.0,
-      "grad_norm": 1.159719705581665,
-      "learning_rate": 2.5768757687576876e-05,
-      "loss": 0.105,
-      "step": 8380
-    },
-    {
-      "epoch": 10.0,
-      "eval_bleu": 0.2555335148067654,
-      "eval_loss": 0.45896273851394653,
-      "eval_rouge1": 0.6440840106866277,
-      "eval_rouge2": 0.3998364161836553,
-      "eval_rougeL": 0.6425824451349154,
-      "eval_runtime": 5.4849,
-      "eval_samples_per_second": 305.385,
-      "eval_steps_per_second": 38.287,
-      "step": 8380
-    },
-    {
-      "epoch": 11.0,
-      "grad_norm": 0.923600435256958,
-      "learning_rate": 2.3191881918819188e-05,
-      "loss": 0.0982,
-      "step": 9218
-    },
-    {
-      "epoch": 11.0,
-      "eval_bleu": 0.25802091175426106,
-      "eval_loss": 0.46457362174987793,
-      "eval_rouge1": 0.6455262594531862,
-      "eval_rouge2": 0.40187791063551404,
-      "eval_rougeL": 0.6445205006389869,
-      "eval_runtime": 6.4432,
-      "eval_samples_per_second": 259.965,
-      "eval_steps_per_second": 32.593,
-      "step": 9218
-    },
-    {
-      "epoch": 12.0,
-      "grad_norm": 0.9120431542396545,
-      "learning_rate": 2.0615006150061504e-05,
-      "loss": 0.0936,
-      "step": 10056
-    },
-    {
-      "epoch": 12.0,
-      "eval_bleu": 0.257202175451904,
-      "eval_loss": 0.470233678817749,
-      "eval_rouge1": 0.6456936789584837,
-      "eval_rouge2": 0.4045772651416589,
-      "eval_rougeL": 0.6445820832270409,
-      "eval_runtime": 13.7399,
-      "eval_samples_per_second": 121.907,
-      "eval_steps_per_second": 15.284,
-      "step": 10056
-    },
-    {
-      "epoch": 13.0,
-      "grad_norm": 0.6435267329216003,
-      "learning_rate": 1.8038130381303812e-05,
-      "loss": 0.0899,
-      "step": 10894
-    },
-    {
-      "epoch": 13.0,
-      "eval_bleu": 0.25765634141938715,
-      "eval_loss": 0.47367072105407715,
-      "eval_rouge1": 0.6488279078084271,
-      "eval_rouge2": 0.40534740561340493,
-      "eval_rougeL": 0.6478436883847489,
-      "eval_runtime": 17.2761,
-      "eval_samples_per_second": 96.955,
-      "eval_steps_per_second": 12.156,
-      "step": 10894
-    },
-    {
-      "epoch": 14.0,
-      "grad_norm": 0.7371336221694946,
-      "learning_rate": 1.5461254612546124e-05,
-      "loss": 0.0871,
-      "step": 11732
-    },
-    {
-      "epoch": 14.0,
-      "eval_bleu": 0.2606034683461693,
-      "eval_loss": 0.4779162108898163,
-      "eval_rouge1": 0.6491650390141908,
-      "eval_rouge2": 0.40618585329089035,
-      "eval_rougeL": 0.6482221321240169,
-      "eval_runtime": 5.371,
-      "eval_samples_per_second": 311.859,
-      "eval_steps_per_second": 39.099,
-      "step": 11732
-    },
-    {
-      "epoch": 14.0,
-      "step": 11732,
-      "total_flos": 6127298150400000.0,
-      "train_loss": 0.03383859399638808,
-      "train_runtime": 1080.9024,
-      "train_samples_per_second": 123.97,
-      "train_steps_per_second": 15.506
+      "epoch": 9.0,
+      "step": 7524,
+      "total_flos": 3930158776320000.0,
+      "train_loss": 0.40989596332913064,
+      "train_runtime": 1923.2663,
+      "train_samples_per_second": 69.517,
+      "train_steps_per_second": 8.694
     }
   ],
   "logging_steps": 500,
-  "max_steps": 16760,
+  "max_steps": 16720,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 20,
   "save_steps": 500,
@@ -310,7 +215,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 6127298150400000.0,
+  "total_flos": 3930158776320000.0,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null