Update README.md

added Model-Description

README.md

@@ -5,6 +5,8 @@ base_model: meta-llama/Meta-Llama-3-8B-Instruct
 tags:
 - axolotl
 - generated_from_trainer
+- spectrum finetuning
+- Deepspeed MultiGPU
 model-index:
 - name: Llama-3-8B-spectrum-25
   results: []
@@ -13,194 +15,23 @@ model-index:
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-[<img src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/axolotl-ai-cloud/axolotl)
-<details><summary>See axolotl config</summary>
-
-axolotl version: `0.4.1`
-```yaml
-base_model: meta-llama/Meta-Llama-3-8B-Instruct
-model_type: LlamaForCausalLM
-tokenizer_type: AutoTokenizer
-is_llama_derived_model: true
-tokenizer_use_fast: true
-hub_model_id: Llama-3-8B-spectrum-25
-
-# load_in_8bit: true
-# load_in_4bit: false
-# strict: false
-
-datasets:
-  - path: yuvraj17/finetune_alpaca_1K
-    type: alpaca
-
-dataset_prepared_path: last_run_prepared
-val_set_size: 0.02
-output_dir: ./outputs/llama-3-8b-spectrum-25
-
-sequence_len: 2048
-sample_packing: false
-eval_sample_packing: false
-pad_to_sequence_len: true
-
-# Model Layers for Llama-3-8B-Instruct (Spectrum with snr values (25%)):
-unfrozen_parameters:
-  - ^lm_head.weight$
-  - ^model.embed_tokens.weight$
-  # input_layernorm layers
-  - model.layers.0.input_layernorm
-  - model.layers.1.input_layernorm
-  - model.layers.2.input_layernorm
-  - model.layers.3.input_layernorm
-  - model.layers.4.input_layernorm
-  - model.layers.5.input_layernorm
-  - model.layers.6.input_layernorm
-  - model.layers.7.input_layernorm
-  # lm_head layers
-  # mlp.down_proj layers
-  - model.layers.1.mlp.down_proj
-  - model.layers.0.mlp.down_proj
-  - model.layers.2.mlp.down_proj
-  - model.layers.30.mlp.down_proj
-  - model.layers.22.mlp.down_proj
-  - model.layers.21.mlp.down_proj
-  - model.layers.5.mlp.down_proj
-  - model.layers.29.mlp.down_proj
-  # mlp.gate_proj layers
-  - model.layers.1.mlp.gate_proj
-  - model.layers.2.mlp.gate_proj
-  - model.layers.3.mlp.gate_proj
-  - model.layers.0.mlp.gate_proj
-  - model.layers.4.mlp.gate_proj
-  - model.layers.25.mlp.gate_proj
-  - model.layers.26.mlp.gate_proj
-  - model.layers.5.mlp.gate_proj
-  # mlp.up_proj layers
-  - model.layers.4.mlp.up_proj
-  - model.layers.0.mlp.up_proj
-  - model.layers.3.mlp.up_proj
-  - model.layers.5.mlp.up_proj
-  - model.layers.7.mlp.up_proj
-  - model.layers.6.mlp.up_proj
-  - model.layers.2.mlp.up_proj
-  - model.layers.1.mlp.up_proj
-  # model.embed_tokens layers
-  # model.norm layers
-  # post_attention_layernorm layers
-  - model.layers.0.post_attention_layernorm
-  - model.layers.1.post_attention_layernorm
-  - model.layers.2.post_attention_layernorm
-  - model.layers.3.post_attention_layernorm
-  - model.layers.4.post_attention_layernorm
-  - model.layers.5.post_attention_layernorm
-  - model.layers.6.post_attention_layernorm
-  - model.layers.7.post_attention_layernorm
-  # self_attn.k_proj layers
-  - model.layers.29.self_attn.k_proj
-  - model.layers.25.self_attn.k_proj
-  - model.layers.23.self_attn.k_proj
-  - model.layers.28.self_attn.k_proj
-  - model.layers.21.self_attn.k_proj
-  - model.layers.19.self_attn.k_proj
-  - model.layers.22.self_attn.k_proj
-  - model.layers.20.self_attn.k_proj
-  # self_attn.o_proj layers
-  - model.layers.14.self_attn.o_proj
-  - model.layers.7.self_attn.o_proj
-  - model.layers.5.self_attn.o_proj
-  - model.layers.11.self_attn.o_proj
-  - model.layers.9.self_attn.o_proj
-  - model.layers.6.self_attn.o_proj
-  - model.layers.13.self_attn.o_proj
-  - model.layers.10.self_attn.o_proj
-  # self_attn.q_proj layers
-  - model.layers.13.self_attn.q_proj
-  - model.layers.9.self_attn.q_proj
-  - model.layers.10.self_attn.q_proj
-  - model.layers.8.self_attn.q_proj
-  - model.layers.14.self_attn.q_proj
-  - model.layers.11.self_attn.q_proj
-  - model.layers.0.self_attn.q_proj
-  - model.layers.15.self_attn.q_proj
-  # self_attn.v_proj layers
-  - model.layers.26.self_attn.v_proj
-  - model.layers.17.self_attn.v_proj
-  - model.layers.28.self_attn.v_proj
-  - model.layers.3.self_attn.v_proj
-  - model.layers.29.self_attn.v_proj
-  - model.layers.21.self_attn.v_proj
-  - model.layers.16.self_attn.v_proj
-  - model.layers.15.self_attn.v_proj
-
-# adapter: lora
-# lora_model_dir:
-# lora_r: 32
-# lora_alpha: 16
-# lora_dropout: 0.05
-# lora_target_linear: true
-# lora_fan_in_fan_out:
-
-wandb_project: llama-3-8B-spectrum
-wandb_entity:
-wandb_watch:
-wandb_name:
-wandb_log_model:
-
-gradient_accumulation_steps: 4
-micro_batch_size: 4
-num_epochs: 2
-optimizer: paged_adamw_32bit
-lr_scheduler: cosine
-learning_rate: 0.0002
-
-train_on_inputs: false
-group_by_length: false
-bf16: auto
-fp16:
-tf32: false
-
-gradient_checkpointing: true
-early_stopping_patience:
-resume_from_checkpoint:
-local_rank:
-logging_steps: 1
-xformers_attention:
-flash_attention: true
-
-warmup_steps: 100
-eval_steps: 0.01
-save_strategy: epoch
-save_steps:
-debug:
-deepspeed: deepspeed_configs/zero2.json
-weight_decay: 0.1
-fsdp:
-fsdp_config:
-special_tokens:
-  pad_token: "<|end_of_text|>"
-
-```
-
-</details><br>
 
 # Llama-3-8B-spectrum-25
 
-This model is a fine-tuned version of [meta-llama/Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) on the None dataset.
+This model is a fine-tuned version of [meta-llama/Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) on the [yuvraj17/finetune_alpaca_1K](https://huggingface.co/datasets/yuvraj17/finetune_alpaca_1K) dataset.
 It achieves the following results on the evaluation set:
 - Loss: 1.2791
 
-## Model description
+## Spectrum Fine-tuning:
+I have used the **Spectrum Fine-tuning** method as described in [Eric Hartford et. al 2024](https://arxiv.org/abs/2406.06623), which selectively targets some ***t%*** of the model layers with the highest **Signal-to-Noise Ratio (SNR)**. By focusing on the most information-dense layers, this approach maximizes fine-tuning efficiency while minimizing compute resources.
+The key goal of Spectrum Fine-tuning is: *minimize the memory footprint and accelerate LLM training without sacrificing performance.*
 
-More information needed
+The 25% layer selection ensures minimal computational overhead for fine-tuning.
 
-## Intended uses & limitations
+## Training:
+- Trained on **2x A40s (48GB VRAM each)** for over 1 hour using the **Axolotl**.
+- Fine-tuning aimed to optimize the balance between model performance and resource efficiency, demonstrating how targeted spectrum fine-tuning can deliver substantial improvements without the need for full-scale model adjustments.
 
-More information needed
-
-## Training and evaluation data
-
-More information needed
-
-## Training procedure
 
 ### Training hyperparameters
 
@@ -219,75 +50,16 @@ The following hyperparameters were used during training:
 - lr_scheduler_warmup_steps: 100
 - num_epochs: 2
 
-### Training results
 
-| Training Loss | Epoch  | Step | Validation Loss |
-|:-------------:|:------:|:----:|:---------------:|
-| 1.4618        | 0.0325 | 1    | 1.2057          |
-| 1.2189        | 0.0650 | 2    | 1.1976          |
-| 1.0899        | 0.0976 | 3    | 1.1611          |
-| 1.2787        | 0.1301 | 4    | 1.1385          |
-| 1.1341        | 0.1626 | 5    | 1.1368          |
-| 1.2793        | 0.1951 | 6    | 1.1228          |
-| 1.2094        | 0.2276 | 7    | 1.1123          |
-| 1.3289        | 0.2602 | 8    | 1.1126          |
-| 1.1179        | 0.2927 | 9    | 1.1123          |
-| 1.2456        | 0.3252 | 10   | 1.1109          |
-| 1.2253        | 0.3577 | 11   | 1.1083          |
-| 1.2563        | 0.3902 | 12   | 1.1079          |
-| 1.3222        | 0.4228 | 13   | 1.1059          |
-| 1.2197        | 0.4553 | 14   | 1.1080          |
-| 1.1862        | 0.4878 | 15   | 1.1054          |
-| 1.1136        | 0.5203 | 16   | 1.1040          |
-| 1.2221        | 0.5528 | 17   | 1.1040          |
-| 1.4475        | 0.5854 | 18   | 1.1049          |
-| 1.0187        | 0.6179 | 19   | 1.1054          |
-| 1.0596        | 0.6504 | 20   | 1.1057          |
-| 1.2075        | 0.6829 | 21   | 1.1063          |
-| 1.0671        | 0.7154 | 22   | 1.1062          |
-| 1.2115        | 0.7480 | 23   | 1.1059          |
-| 1.1137        | 0.7805 | 24   | 1.1061          |
-| 1.5483        | 0.8130 | 25   | 1.1097          |
-| 1.369         | 0.8455 | 26   | 1.1120          |
-| 1.0528        | 0.8780 | 27   | 1.1155          |
-| 1.2126        | 0.9106 | 28   | 1.1169          |
-| 1.0164        | 0.9431 | 29   | 1.1167          |
-| 1.2082        | 0.9756 | 30   | 1.1183          |
-| 1.0256        | 1.0081 | 31   | 1.1191          |
-| 0.6859        | 1.0407 | 32   | 1.1267          |
-| 0.722         | 1.0732 | 33   | 1.1505          |
-| 0.7161        | 1.1057 | 34   | 1.1719          |
-| 0.724         | 1.1382 | 35   | 1.1829          |
-| 0.67          | 1.1707 | 36   | 1.1844          |
-| 0.5737        | 1.2033 | 37   | 1.1874          |
-| 0.7081        | 1.2358 | 38   | 1.1940          |
-| 0.7239        | 1.2683 | 39   | 1.1978          |
-| 0.5927        | 1.3008 | 40   | 1.2022          |
-| 0.6079        | 1.3333 | 41   | 1.2070          |
-| 0.6427        | 1.3659 | 42   | 1.2104          |
-| 0.506         | 1.3984 | 43   | 1.2134          |
-| 0.4582        | 1.4309 | 44   | 1.2195          |
-| 0.7492        | 1.4634 | 45   | 1.2208          |
-| 0.538         | 1.4959 | 46   | 1.2258          |
-| 0.7147        | 1.5285 | 47   | 1.2299          |
-| 0.6565        | 1.5610 | 48   | 1.2339          |
-| 0.8011        | 1.5935 | 49   | 1.2365          |
-| 0.6986        | 1.6260 | 50   | 1.2396          |
-| 0.7924        | 1.6585 | 51   | 1.2472          |
-| 0.8128        | 1.6911 | 52   | 1.2542          |
-| 0.6733        | 1.7236 | 53   | 1.2616          |
-| 0.7363        | 1.7561 | 54   | 1.2693          |
-| 0.5815        | 1.7886 | 55   | 1.2762          |
-| 0.6571        | 1.8211 | 56   | 1.2750          |
-| 0.6985        | 1.8537 | 57   | 1.2748          |
-| 0.7519        | 1.8862 | 58   | 1.2715          |
-| 0.8171        | 1.9187 | 59   | 1.2733          |
-| 0.7373        | 1.9512 | 60   | 1.2791          |
+![Train/loss Curve Image](https://cdn-uploads.huggingface.co/production/uploads/66137d95e8d2cda230ddcea6/eSBh0SmeGYYUfx9pKgMIv.png)
+
+![eval/loss Curve Image](https://cdn-uploads.huggingface.co/production/uploads/66137d95e8d2cda230ddcea6/xNslkLH1pKot7tzWtIiu9.png)
 
 
 ### Framework versions
 
+- Axolotl 0.4.0
 - Transformers 4.44.2
 - Pytorch 2.4.0+cu121
 - Datasets 2.20.0
-- Tokenizers 0.19.1
+- Tokenizers 0.19.1