release v1 epoch 1

configs/minotaur.yml

@@ -83,10 +83,6 @@ datasets:
     type: alpaca_chat
   - path: ehartford/WizardLM_alpaca_evol_instruct_70k_unfiltered
     type: alpaca:chat
-  - path: ewof/code-alpaca-instruct-unfiltered
-    type: alpaca_chat
-  - path: teknium/GPT4-LLM-Cleaned
-    type: alpaca_chat
 dataset_prepared_path: last_run_prepared
 val_set_size: 0.01
 adapter:
@@ -105,8 +101,8 @@ wandb_run_id:
 wandb_log_model:
 output_dir: ./minotaur-13b
 gradient_accumulation_steps: 1
-micro_batch_size: 12
-num_epochs: 12
+micro_batch_size: 8
+num_epochs: 16
 optimizer: adamw_bnb_8bit
 torchdistx_path:
 lr_scheduler: cosine
@@ -125,13 +121,13 @@ xformers_attention: true
 flash_attention:
 gptq_groupsize:
 gptq_model_v1:
-warmup_steps: 100
+warmup_steps: 300
 eval_steps: 29
 save_steps: 29
 load_best_model_at_end: false
 debug:
 deepspeed:
-weight_decay: 0.0007
+weight_decay: 0.00003
 fsdp:
   - full_shard
   - auto_wrap
@@ -143,3 +139,4 @@ special_tokens:
   eos_token: "</s>"
   unk_token: "<unk>"
 
+

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0cbf121fdeddf1b6358672ae19bae3d48a992a6b6922a5a3561b95f970fb593d
+oid sha256:8faa183c31365889b8f5de5f719c6a463e82833d2a49063f25c5c24755a4bcaf
 size 26031937261