add new config

configs/minotaur.yml

@@ -83,6 +83,10 @@ datasets:
     type: alpaca_chat
   - path: ehartford/WizardLM_alpaca_evol_instruct_70k_unfiltered
     type: alpaca:chat
+  - path: ewof/code-alpaca-instruct-unfiltered
+    type: alpaca_chat
+  - path: teknium/GPT4-LLM-Cleaned
+    type: alpaca_chat
 dataset_prepared_path: last_run_prepared
 val_set_size: 0.01
 adapter:
@@ -106,9 +110,9 @@ num_epochs: 12
 optimizer: adamw_bnb_8bit
 torchdistx_path:
 lr_scheduler: cosine
-learning_rate: 0.00001
+learning_rate: 0.00005
 train_on_inputs: false
-group_by_length: false
+group_by_length: true
 bf16: true
 fp16: false
 tf32: true
@@ -122,13 +126,12 @@ flash_attention:
 gptq_groupsize:
 gptq_model_v1:
 warmup_steps: 100
-eval_steps: 27
-save_steps: 27
+eval_steps: 29
+save_steps: 29
 load_best_model_at_end: false
-save_total_limit: 10
 debug:
 deepspeed:
-weight_decay: 0.00001
+weight_decay: 0.0007
 fsdp:
   - full_shard
   - auto_wrap