Upload training_config.yml with huggingface_hub

training_config.yml

@@ -19,17 +19,17 @@ checkpointer:
   - meta_model_0.pt
   adapter_checkpoint: null
   recipe_checkpoint: null
-  output_dir: output_checkpoints/experiment_2
+  output_dir: output_checkpoints/experiment_5
   model_type: LLAMA3
 resume_from_checkpoint: false
-interim_checkpoint_steps: 200
+interim_checkpoint_steps: 15000
 interim_gen_steps: null
 max_new_tokens: 77
 temperature: 0.6
 top_k: 231
 dataset:
   _component_: ds.EvenBatcher
-  buffer_size: 36
+  buffer_size: 72
   dataset:
     _component_: ds.RoundRobinDataset
     datasets:
@@ -46,25 +46,25 @@ dataset:
       train_on_input: false
 seed: null
 shuffle: true
-batch_size: 8
+batch_size: 6
 optimizer:
   _component_: torch.optim.AdamW
-  weight_decay: 0.01
-  lr: 1.0e-05
+  weight_decay: 0.008
+  lr: 0.0001
 lr_scheduler:
   _component_: torchtune.modules.get_cosine_schedule_with_warmup
-  num_warmup_steps: 200
+  num_warmup_steps: 100
 loss:
   _component_: torch.nn.CrossEntropyLoss
-epochs: 1
+epochs: 6
 max_steps_per_epoch: null
-gradient_accumulation_steps: 64
+gradient_accumulation_steps: 16
 compile: false
 output_dir: /tmp/lora_finetune_output
 metric_logger:
-  _component_: torchtune.utils.metric_logging.WandBLogger
+  _component_: torchtune.utils.metric_logging.DiskLogger
   log_dir: ${output_dir}
-log_every_n_steps: 100
+log_every_n_steps: null
 device: cuda
 dtype: bf16
 enable_activation_checkpointing: false