Update README.md

README.md

@@ -68,21 +68,15 @@ args = TrainingArguments(
 
     output_dir="/content/gdrive/MyDrive/model/NEW-Meta-Llama-3-8B-MEDAL-flash-attention-2-cosine-evaldata",
 
-    #num_train_epochs=3,                     # number of training epochs
-    num_train_epochs=1,                    # number of training epochs for POC
+    #num_train_epochs=3,                    # number of training epochs
+    num_train_epochs=1,                     # number of training epochs for POC
     per_device_train_batch_size=2,          # batch size per device during training
-    #2
+  
     gradient_accumulation_steps=8,          # number of steps before performing a backward/update pass
     gradient_checkpointing=True,            # use gradient checkpointing to save memory
-    #gradient_checkpointing_kwargs={"use_reentrant": True},
-    optim="adamw_torch_fused",              # use fused adamw optimizer
-
-    #ELECTRA is trained with Adam optimizer with learning
-    #rate of 0.00002 and with batch size of 16
-
-    #trainer = Trainer(model=model, args=training_args, train_dataset=ds, optimizers=(adam_bnb_optim, None))
-    logging_steps=200,                       # log every 10 steps
-    #save_strategy="epoch",                  # save checkpoint every epoch
+    
+    optim="adamw_torch_fused",               # use fused adamw optimizer
+    logging_steps=200,                       # log every 200 steps
 
     learning_rate=2e-4,                     # learning rate, based on QLoRA paper # i used in the first model
     bf16=True,                              # use bfloat16 precision
@@ -95,15 +89,16 @@ args = TrainingArguments(
 
     push_to_hub=True,                       # push model to hub
     report_to="tensorboard",                # report metrics to tensorboard
+    
     gradient_checkpointing_kwargs={"use_reentrant": True},
 
     load_best_model_at_end=True,
     logging_dir="/content/gdrive/MyDrive/model/NEW-Meta-Llama-3-8B-MEDAL-flash-attention-2-cosine-evaldata/logs",
 
-    evaluation_strategy="steps",   # Evaluate at step intervals
+    evaluation_strategy="steps",    # Evaluate at step intervals
     eval_steps=200,                 # Evaluate every 50 steps
-    save_strategy="steps",  # Save checkpoints at step intervals
-    save_steps=200,  # Save every 50 steps (aligned with eval_steps)
+    save_strategy="steps",          # Save checkpoints at step intervals
+    save_steps=200,                 # Save every 50 steps (aligned with eval_steps)
     metric_for_best_model = "loss",
     ]
 )