Update README.md

README.md

@@ -18,90 +18,6 @@ pipeline_tag: text-generation
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-[<img src="https://raw.githubusercontent.com/OpenAccess-AI-Collective/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/OpenAccess-AI-Collective/axolotl)
-<details><summary>See axolotl config</summary>
-
-axolotl version: `0.4.0`
-```yaml
-base_model: MaziyarPanahi/Qwen1.5-8x7b
-model_type: Qwen2ForCausalLM
-tokenizer_type: Qwen2Tokenizer
-
-trust_remote_code: true
-
-hub_model_id: MaziyarPanahi/Qwen1.5-8x7b-v0.1
-hf_use_auth_token: true
-
-load_in_8bit: false
-load_in_4bit: true
-strict: false
-
-
-datasets:
-  - path: Crystalcareai/MoD-150k
-    type: sharegpt
-
-
-dataset_prepared_path:
-val_set_size: 0.05
-output_dir: ./Qwen1.5-8x7b-v0.1-lora-out
-
-model_config:
-  output_router_logits: true
-
-adapter: qlora
-lora_model_dir:
-sequence_len: 2048
-sample_packing: true
-pad_to_sequence_len: true
-
-
-lora_r: 32
-lora_alpha: 16
-lora_dropout: 0.05
-lora_target_linear: true
-lora_fan_in_fan_out:
-
-
-gradient_accumulation_steps: 2
-micro_batch_size: 2
-num_epochs: 1
-optimizer: adamw_bnb_8bit
-lr_scheduler: cosine
-learning_rate: 0.0002
-
-
-train_on_inputs: false
-group_by_length: false
-bf16: auto
-fp16:
-tf32: false
-
-
-gradient_checkpointing: true
-early_stopping_patience: 
-resume_from_checkpoint:
-local_rank:
-logging_steps: 1
-xformers_attention:
-flash_attention: true
-
-
-warmup_steps: 10
-evals_per_epoch: 4
-eval_table_size:
-eval_max_new_tokens: 128
-saves_per_epoch: 1
-debug:
-deepspeed:
-weight_decay: 0.0
-fsdp:
-fsdp_config:
-special_tokens:
-```
-
-</details><br>
-
 # Qwen1.5-8x7b-v0.1
 
 This model is a fine-tuned version of [MaziyarPanahi/Qwen1.5-8x7b](https://huggingface.co/MaziyarPanahi/Qwen1.5-8x7b) on the [Crystalcareai/MoD-150k](https://huggingface.co/datasets/Crystalcareai/MoD-150k) dataset.
@@ -381,4 +297,89 @@ The following hyperparameters were used during training:
 - Transformers 4.39.0.dev0
 - Pytorch 2.2.0+cu121
 - Datasets 2.17.0
-- Tokenizers 0.15.0
+- Tokenizers 0.15.0
+
+[<img src="https://raw.githubusercontent.com/OpenAccess-AI-Collective/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/OpenAccess-AI-Collective/axolotl)
+<details><summary>See axolotl config</summary>
+
+axolotl version: `0.4.0`
+```yaml
+base_model: MaziyarPanahi/Qwen1.5-8x7b
+model_type: Qwen2ForCausalLM
+tokenizer_type: Qwen2Tokenizer
+
+trust_remote_code: true
+
+hub_model_id: MaziyarPanahi/Qwen1.5-8x7b-v0.1
+hf_use_auth_token: true
+
+load_in_8bit: false
+load_in_4bit: true
+strict: false
+
+
+datasets:
+  - path: Crystalcareai/MoD-150k
+    type: sharegpt
+
+
+dataset_prepared_path:
+val_set_size: 0.05
+output_dir: ./Qwen1.5-8x7b-v0.1-lora-out
+
+model_config:
+  output_router_logits: true
+
+adapter: qlora
+lora_model_dir:
+sequence_len: 2048
+sample_packing: true
+pad_to_sequence_len: true
+
+
+lora_r: 32
+lora_alpha: 16
+lora_dropout: 0.05
+lora_target_linear: true
+lora_fan_in_fan_out:
+
+
+gradient_accumulation_steps: 2
+micro_batch_size: 2
+num_epochs: 1
+optimizer: adamw_bnb_8bit
+lr_scheduler: cosine
+learning_rate: 0.0002
+
+
+train_on_inputs: false
+group_by_length: false
+bf16: auto
+fp16:
+tf32: false
+
+
+gradient_checkpointing: true
+early_stopping_patience: 
+resume_from_checkpoint:
+local_rank:
+logging_steps: 1
+xformers_attention:
+flash_attention: true
+
+
+warmup_steps: 10
+evals_per_epoch: 4
+eval_table_size:
+eval_max_new_tokens: 128
+saves_per_epoch: 1
+debug:
+deepspeed:
+weight_decay: 0.0
+fsdp:
+fsdp_config:
+special_tokens:
+```
+
+</details><br>
+-