lazy lora for llama2 13b hf

README.md

@@ -1,3 +1,160 @@
 ---
 license: llama2
 ---
+
+## Lazy LoRA
+
+### Benefits
+
+0. using the updated [Meta's LLaMA-2 models](https://huggingface.co/meta-llama/Llama-2-13b-hf).
+1. support [4-bit qlora](https://arxiv.org/abs/2305.14314), extreme GPU memory and inference time saving;
+2. better MMLU evaluation dataset results, llama2-13b's 54.8% to our 57.16% (+2.36%).
+3. This lazy-lora adapter is based on [Meta's LLaMA-2-13b-hf](https://huggingface.co/meta-llama/Llama-2-13b-hf), and using the [oasst1 dataset](https://huggingface.co/datasets/OpenAssistant/oasst1), following [Guanaco](https://huggingface.co/timdettmers/guanaco-65b).
+
+### Introduction
+Determine the rank of LoRA layers by the singular values of pretrained weight matrices.
+Also, combines:
+1. LoRA: [LORA: LOW-RANK ADAPTATION OF LARGE LANGUAGE MODELS](https://arxiv.org/abs/2106.09685)
+2. Prefix Tuning: [Prefix-Tuning: Optimizing Continuous Prompts for Generation](https://aclanthology.org/2021.acl-long.3
+53/), [P-Tuning v2: Prompt Tuning Can Be Comparable to Fine-tuning Universally Across Scales and Tasks](https://arxiv.or
+g/pdf/2110.07602.pdf)
+3. Prompt Tuning: [The Power of Scale for Parameter-Efficient Prompt Tuning](https://arxiv.org/abs/2104.08691)
+4. LLaMA adapter: [LLaMA-Adapter: Efficient Fine-tuning of Language Models with Zero-init Attention] (https://arxiv.org/abs/2303.16199)
+in one model.
+
+This allows you to perform LoRA (additional low rank adapters inserted to each linear layer), and prompt learning (additional virtual tokens attached to the input and to the attention layers acting as `past_key_values`)
+
+## Usage:
+```python
+import sys
+sys.path.insert(1, '/workspace/asr/peft/src')
+# TODO set this path to the lazy-lora source code path, 
+# or you can install it from source code:
+# TODO, please install lazylora for usage:
+# git clone git@github.com:Xianchao-Wu/peft.git
+# cd peft
+# python setup.py install
+
+from transformers import (AutoTokenizer, 
+	AutoModelForCausalLM, BitsAndBytesConfig)
+from peft import PeftModel, PeftConfig
+import os
+import torch
+
+#import ipdb; ipdb.set_trace()
+cache_dir="/workspace/asr/peft/qlora"
+# TODO set this cache_dir to the path where you 
+# stored (or, want to store) llama2-13b-hf model
+
+lazylora_dir=os.getcwd() 
+# the path that contains 'adapter_config.json' 
+#     and 'adapter_model.bin'
+
+config = PeftConfig.from_pretrained(lazylora_dir)
+
+tokenizer = AutoTokenizer.from_pretrained(
+    config.base_model_name_or_path,
+    cache_dir=cache_dir,
+    use_auth_token=True
+)
+
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type='nf4',
+    bnb_4bit_compute_dtype=torch.bfloat16
+)
+
+model = AutoModelForCausalLM.from_pretrained(
+    config.base_model_name_or_path,
+    quantization_config=bnb_config,
+    device_map="auto",
+    cache_dir=cache_dir,
+    use_auth_token=True
+)
+#model.print_trainable_parameters()
+print(sum(p.numel() for p in model.parameters()))
+# 6,671,979,520 -> half-size of 13B due to 4-bit loading
+
+model = PeftModel.from_pretrained(model, lazylora_dir)
+print('after adding lazy lora parameters:')
+model.print_trainable_parameters()
+# trainable params: 0 || all params: 6,922,290,688 || trainable%: 0.0
+```
+
+## MMLU result:
+
+```json
+{"mmlu_loss": 1.7513423996918502,
+ "mmlu_eval_accuracy_high_school_microeconomics": 0.6538461538461539,
+ "mmlu_eval_accuracy_human_aging": 0.6956521739130435,
+ "mmlu_eval_accuracy_high_school_biology": 0.5625,
+ "mmlu_eval_accuracy_machine_learning": 0.45454545454545453,
+ "mmlu_eval_accuracy_moral_scenarios": 0.31,
+ "mmlu_eval_accuracy_astronomy": 0.6875,
+ "mmlu_eval_accuracy_medical_genetics": 0.8181818181818182,
+ "mmlu_eval_accuracy_high_school_government_and_politics": 0.7142857142857143,
+ "mmlu_eval_accuracy_anatomy": 0.35714285714285715,
+ "mmlu_eval_accuracy_prehistory": 0.5714285714285714,
+ "mmlu_eval_accuracy_high_school_mathematics": 0.20689655172413793,
+ "mmlu_eval_accuracy_philosophy": 0.6176470588235294,
+ "mmlu_eval_accuracy_high_school_computer_science": 0.5555555555555556,
+ "mmlu_eval_accuracy_high_school_statistics": 0.30434782608695654,
+ "mmlu_eval_accuracy_conceptual_physics": 0.38461538461538464,
+ "mmlu_eval_accuracy_global_facts": 0.5,
+ "mmlu_eval_accuracy_high_school_us_history": 0.7727272727272727,
+ "mmlu_eval_accuracy_professional_accounting": 0.41935483870967744,
+ "mmlu_eval_accuracy_college_computer_science": 0.5454545454545454,
+ "mmlu_eval_accuracy_econometrics": 0.4166666666666667,
+ "mmlu_eval_accuracy_high_school_physics": 0.23529411764705882,
+ "mmlu_eval_accuracy_public_relations": 0.6666666666666666,
+ "mmlu_eval_accuracy_us_foreign_policy": 0.9090909090909091,
+ "mmlu_eval_accuracy_miscellaneous": 0.6744186046511628,
+ "mmlu_eval_accuracy_college_mathematics": 0.45454545454545453,
+ "mmlu_eval_accuracy_management": 0.7272727272727273,
+ "mmlu_eval_accuracy_college_biology": 0.625,
+ "mmlu_eval_accuracy_high_school_world_history": 0.5384615384615384,
+ "mmlu_eval_accuracy_electrical_engineering": 0.5,
+ "mmlu_eval_accuracy_computer_security": 0.8181818181818182,
+ "mmlu_eval_accuracy_clinical_knowledge": 0.4482758620689655,
+ "mmlu_eval_accuracy_professional_psychology": 0.5507246376811594,
+ "mmlu_eval_accuracy_high_school_geography": 0.8181818181818182,
+ "mmlu_eval_accuracy_high_school_psychology": 0.8333333333333334,
+ "mmlu_eval_accuracy_moral_disputes": 0.631578947368421,
+ "mmlu_eval_accuracy_formal_logic": 0.2857142857142857,
+ "mmlu_eval_accuracy_international_law": 0.8461538461538461,
+ "mmlu_eval_accuracy_nutrition": 0.7878787878787878,
+ "mmlu_eval_accuracy_marketing": 0.88,
+ "mmlu_eval_accuracy_high_school_chemistry": 0.4090909090909091,
+ "mmlu_eval_accuracy_college_chemistry": 0.375,
+ "mmlu_eval_accuracy_professional_medicine": 0.5483870967741935,
+ "mmlu_eval_accuracy_virology": 0.3888888888888889,
+ "mmlu_eval_accuracy_logical_fallacies": 0.7222222222222222,
+ "mmlu_eval_accuracy_high_school_european_history": 0.7777777777777778,
+ "mmlu_eval_accuracy_jurisprudence": 0.6363636363636364,
+ "mmlu_eval_accuracy_human_sexuality": 0.5833333333333334,
+ "mmlu_eval_accuracy_abstract_algebra": 0.2727272727272727,
+ "mmlu_eval_accuracy_business_ethics": 0.2727272727272727,
+ "mmlu_eval_accuracy_security_studies": 0.7037037037037037,
+ "mmlu_eval_accuracy_professional_law": 0.4,
+ "mmlu_eval_accuracy_college_medicine": 0.5,
+ "mmlu_eval_accuracy_elementary_mathematics": 0.34146341463414637,
+ "mmlu_eval_accuracy_high_school_macroeconomics": 0.5813953488372093,
+ "mmlu_eval_accuracy_sociology": 0.8636363636363636,
+ "mmlu_eval_accuracy_world_religions": 0.7894736842105263,
+ "mmlu_eval_accuracy_college_physics": 0.6363636363636364,
+ "mmlu_eval_accuracy": 0.5716083571911647,
+ "epoch": 0.68}
+```
+
+## License and intended use 
+
+This lazy-lora adapter is based on [Meta's LLaMA-2-13b-hf](https://huggingface.co/meta-llama/Llama-2-13b-hf), and using the [oasst1 dataset](https://huggingface.co/datasets/OpenAssistant/oasst1), following [Guanaco](https://huggingface.co/timdettmers/guanaco-65b).
+
+lazy lora adapter weights are available under LLAMA-2 license. Note the use of the lazy lora adapter weights, requires access to the LLaMA model weighs. Lazy lora is based on LLaMA and therefore should be used according to the LLaMA license.
+
+
+## Risks and Biases
+
+The model can produce factually incorrect output, and should not be relied on to produce factually accurate information. The model was trained on various public datasets; it is possible that this model could generate lewd, biased, or otherwise offensive outputs.
+

adapter_config.json

@@ -0,0 +1,316 @@
+{
+  "base_model_name_or_path": "meta-llama/Llama-2-13b-hf",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lazy_lora_weights": true,
+  "is_r_by_svd": true,
+  "is_r_reuse": true,
+  "lazy_lora_alpha": 16.0,
+  "lazy_lora_dropout": 0.05,
+  "lazy_pre_adapter_type": "none",
+  "lazy_pre_lora_alpha": 0.1,
+  "modules_to_save": null,
+  "num_attention_heads": 40,
+  "num_layers": 40,
+  "num_transformer_submodules": 1,
+  "num_virtual_tokens": null,
+  "peft_type": "LAZY_LORA",
+  "prefix_tuning_config": null,
+  "prompt_tuning_config": null,
+  "r": 64,
+  "r_by_module_dict": {
+    "model.layers.0.mlp.down_proj": 56,
+    "model.layers.0.mlp.gate_proj": 45,
+    "model.layers.0.mlp.up_proj": 47,
+    "model.layers.0.self_attn.k_proj": 14,
+    "model.layers.0.self_attn.o_proj": 24,
+    "model.layers.0.self_attn.q_proj": 13,
+    "model.layers.0.self_attn.v_proj": 30,
+    "model.layers.1.mlp.down_proj": 61,
+    "model.layers.1.mlp.gate_proj": 51,
+    "model.layers.1.mlp.up_proj": 55,
+    "model.layers.1.self_attn.k_proj": 39,
+    "model.layers.1.self_attn.o_proj": 36,
+    "model.layers.1.self_attn.q_proj": 39,
+    "model.layers.1.self_attn.v_proj": 38,
+    "model.layers.10.mlp.down_proj": 62,
+    "model.layers.10.mlp.gate_proj": 63,
+    "model.layers.10.mlp.up_proj": 64,
+    "model.layers.10.self_attn.k_proj": 68,
+    "model.layers.10.self_attn.o_proj": 56,
+    "model.layers.10.self_attn.q_proj": 69,
+    "model.layers.10.self_attn.v_proj": 55,
+    "model.layers.11.mlp.down_proj": 63,
+    "model.layers.11.mlp.gate_proj": 62,
+    "model.layers.11.mlp.up_proj": 64,
+    "model.layers.11.self_attn.k_proj": 71,
+    "model.layers.11.self_attn.o_proj": 55,
+    "model.layers.11.self_attn.q_proj": 70,
+    "model.layers.11.self_attn.v_proj": 55,
+    "model.layers.12.mlp.down_proj": 63,
+    "model.layers.12.mlp.gate_proj": 62,
+    "model.layers.12.mlp.up_proj": 65,
+    "model.layers.12.self_attn.k_proj": 68,
+    "model.layers.12.self_attn.o_proj": 56,
+    "model.layers.12.self_attn.q_proj": 67,
+    "model.layers.12.self_attn.v_proj": 57,
+    "model.layers.13.mlp.down_proj": 64,
+    "model.layers.13.mlp.gate_proj": 62,
+    "model.layers.13.mlp.up_proj": 65,
+    "model.layers.13.self_attn.k_proj": 64,
+    "model.layers.13.self_attn.o_proj": 58,
+    "model.layers.13.self_attn.q_proj": 65,
+    "model.layers.13.self_attn.v_proj": 58,
+    "model.layers.14.mlp.down_proj": 64,
+    "model.layers.14.mlp.gate_proj": 62,
+    "model.layers.14.mlp.up_proj": 65,
+    "model.layers.14.self_attn.k_proj": 68,
+    "model.layers.14.self_attn.o_proj": 58,
+    "model.layers.14.self_attn.q_proj": 68,
+    "model.layers.14.self_attn.v_proj": 58,
+    "model.layers.15.mlp.down_proj": 65,
+    "model.layers.15.mlp.gate_proj": 62,
+    "model.layers.15.mlp.up_proj": 66,
+    "model.layers.15.self_attn.k_proj": 67,
+    "model.layers.15.self_attn.o_proj": 59,
+    "model.layers.15.self_attn.q_proj": 66,
+    "model.layers.15.self_attn.v_proj": 60,
+    "model.layers.16.mlp.down_proj": 65,
+    "model.layers.16.mlp.gate_proj": 62,
+    "model.layers.16.mlp.up_proj": 66,
+    "model.layers.16.self_attn.k_proj": 67,
+    "model.layers.16.self_attn.o_proj": 60,
+    "model.layers.16.self_attn.q_proj": 66,
+    "model.layers.16.self_attn.v_proj": 61,
+    "model.layers.17.mlp.down_proj": 65,
+    "model.layers.17.mlp.gate_proj": 63,
+    "model.layers.17.mlp.up_proj": 65,
+    "model.layers.17.self_attn.k_proj": 67,
+    "model.layers.17.self_attn.o_proj": 63,
+    "model.layers.17.self_attn.q_proj": 67,
+    "model.layers.17.self_attn.v_proj": 62,
+    "model.layers.18.mlp.down_proj": 65,
+    "model.layers.18.mlp.gate_proj": 63,
+    "model.layers.18.mlp.up_proj": 65,
+    "model.layers.18.self_attn.k_proj": 69,
+    "model.layers.18.self_attn.o_proj": 65,
+    "model.layers.18.self_attn.q_proj": 69,
+    "model.layers.18.self_attn.v_proj": 65,
+    "model.layers.19.mlp.down_proj": 65,
+    "model.layers.19.mlp.gate_proj": 64,
+    "model.layers.19.mlp.up_proj": 65,
+    "model.layers.19.self_attn.k_proj": 66,
+    "model.layers.19.self_attn.o_proj": 65,
+    "model.layers.19.self_attn.q_proj": 66,
+    "model.layers.19.self_attn.v_proj": 64,
+    "model.layers.2.mlp.down_proj": 62,
+    "model.layers.2.mlp.gate_proj": 59,
+    "model.layers.2.mlp.up_proj": 60,
+    "model.layers.2.self_attn.k_proj": 60,
+    "model.layers.2.self_attn.o_proj": 50,
+    "model.layers.2.self_attn.q_proj": 60,
+    "model.layers.2.self_attn.v_proj": 49,
+    "model.layers.20.mlp.down_proj": 65,
+    "model.layers.20.mlp.gate_proj": 64,
+    "model.layers.20.mlp.up_proj": 65,
+    "model.layers.20.self_attn.k_proj": 67,
+    "model.layers.20.self_attn.o_proj": 65,
+    "model.layers.20.self_attn.q_proj": 68,
+    "model.layers.20.self_attn.v_proj": 64,
+    "model.layers.21.mlp.down_proj": 65,
+    "model.layers.21.mlp.gate_proj": 64,
+    "model.layers.21.mlp.up_proj": 65,
+    "model.layers.21.self_attn.k_proj": 65,
+    "model.layers.21.self_attn.o_proj": 67,
+    "model.layers.21.self_attn.q_proj": 65,
+    "model.layers.21.self_attn.v_proj": 66,
+    "model.layers.22.mlp.down_proj": 65,
+    "model.layers.22.mlp.gate_proj": 65,
+    "model.layers.22.mlp.up_proj": 65,
+    "model.layers.22.self_attn.k_proj": 68,
+    "model.layers.22.self_attn.o_proj": 71,
+    "model.layers.22.self_attn.q_proj": 68,
+    "model.layers.22.self_attn.v_proj": 70,
+    "model.layers.23.mlp.down_proj": 65,
+    "model.layers.23.mlp.gate_proj": 65,
+    "model.layers.23.mlp.up_proj": 64,
+    "model.layers.23.self_attn.k_proj": 65,
+    "model.layers.23.self_attn.o_proj": 71,
+    "model.layers.23.self_attn.q_proj": 66,
+    "model.layers.23.self_attn.v_proj": 69,
+    "model.layers.24.mlp.down_proj": 65,
+    "model.layers.24.mlp.gate_proj": 66,
+    "model.layers.24.mlp.up_proj": 64,
+    "model.layers.24.self_attn.k_proj": 67,
+    "model.layers.24.self_attn.o_proj": 71,
+    "model.layers.24.self_attn.q_proj": 67,
+    "model.layers.24.self_attn.v_proj": 70,
+    "model.layers.25.mlp.down_proj": 65,
+    "model.layers.25.mlp.gate_proj": 66,
+    "model.layers.25.mlp.up_proj": 64,
+    "model.layers.25.self_attn.k_proj": 66,
+    "model.layers.25.self_attn.o_proj": 73,
+    "model.layers.25.self_attn.q_proj": 67,
+    "model.layers.25.self_attn.v_proj": 72,
+    "model.layers.26.mlp.down_proj": 65,
+    "model.layers.26.mlp.gate_proj": 66,
+    "model.layers.26.mlp.up_proj": 64,
+    "model.layers.26.self_attn.k_proj": 67,
+    "model.layers.26.self_attn.o_proj": 73,
+    "model.layers.26.self_attn.q_proj": 67,
+    "model.layers.26.self_attn.v_proj": 73,
+    "model.layers.27.mlp.down_proj": 65,
+    "model.layers.27.mlp.gate_proj": 66,
+    "model.layers.27.mlp.up_proj": 65,
+    "model.layers.27.self_attn.k_proj": 68,
+    "model.layers.27.self_attn.o_proj": 73,
+    "model.layers.27.self_attn.q_proj": 68,
+    "model.layers.27.self_attn.v_proj": 72,
+    "model.layers.28.mlp.down_proj": 65,
+    "model.layers.28.mlp.gate_proj": 67,
+    "model.layers.28.mlp.up_proj": 65,
+    "model.layers.28.self_attn.k_proj": 67,
+    "model.layers.28.self_attn.o_proj": 73,
+    "model.layers.28.self_attn.q_proj": 68,
+    "model.layers.28.self_attn.v_proj": 73,
+    "model.layers.29.mlp.down_proj": 65,
+    "model.layers.29.mlp.gate_proj": 67,
+    "model.layers.29.mlp.up_proj": 65,
+    "model.layers.29.self_attn.k_proj": 69,
+    "model.layers.29.self_attn.o_proj": 74,
+    "model.layers.29.self_attn.q_proj": 69,
+    "model.layers.29.self_attn.v_proj": 73,
+    "model.layers.3.mlp.down_proj": 62,
+    "model.layers.3.mlp.gate_proj": 63,
+    "model.layers.3.mlp.up_proj": 62,
+    "model.layers.3.self_attn.k_proj": 66,
+    "model.layers.3.self_attn.o_proj": 52,
+    "model.layers.3.self_attn.q_proj": 64,
+    "model.layers.3.self_attn.v_proj": 53,
+    "model.layers.30.mlp.down_proj": 65,
+    "model.layers.30.mlp.gate_proj": 67,
+    "model.layers.30.mlp.up_proj": 65,
+    "model.layers.30.self_attn.k_proj": 68,
+    "model.layers.30.self_attn.o_proj": 77,
+    "model.layers.30.self_attn.q_proj": 69,
+    "model.layers.30.self_attn.v_proj": 76,
+    "model.layers.31.mlp.down_proj": 65,
+    "model.layers.31.mlp.gate_proj": 67,
+    "model.layers.31.mlp.up_proj": 65,
+    "model.layers.31.self_attn.k_proj": 69,
+    "model.layers.31.self_attn.o_proj": 74,
+    "model.layers.31.self_attn.q_proj": 69,
+    "model.layers.31.self_attn.v_proj": 74,
+    "model.layers.32.mlp.down_proj": 65,
+    "model.layers.32.mlp.gate_proj": 67,
+    "model.layers.32.mlp.up_proj": 66,
+    "model.layers.32.self_attn.k_proj": 66,
+    "model.layers.32.self_attn.o_proj": 77,
+    "model.layers.32.self_attn.q_proj": 67,
+    "model.layers.32.self_attn.v_proj": 76,
+    "model.layers.33.mlp.down_proj": 65,
+    "model.layers.33.mlp.gate_proj": 67,
+    "model.layers.33.mlp.up_proj": 66,
+    "model.layers.33.self_attn.k_proj": 66,
+    "model.layers.33.self_attn.o_proj": 75,
+    "model.layers.33.self_attn.q_proj": 67,
+    "model.layers.33.self_attn.v_proj": 75,
+    "model.layers.34.mlp.down_proj": 65,
+    "model.layers.34.mlp.gate_proj": 67,
+    "model.layers.34.mlp.up_proj": 67,
+    "model.layers.34.self_attn.k_proj": 64,
+    "model.layers.34.self_attn.o_proj": 77,
+    "model.layers.34.self_attn.q_proj": 65,
+    "model.layers.34.self_attn.v_proj": 77,
+    "model.layers.35.mlp.down_proj": 66,
+    "model.layers.35.mlp.gate_proj": 67,
+    "model.layers.35.mlp.up_proj": 67,
+    "model.layers.35.self_attn.k_proj": 64,
+    "model.layers.35.self_attn.o_proj": 77,
+    "model.layers.35.self_attn.q_proj": 64,
+    "model.layers.35.self_attn.v_proj": 76,
+    "model.layers.36.mlp.down_proj": 67,
+    "model.layers.36.mlp.gate_proj": 67,
+    "model.layers.36.mlp.up_proj": 67,
+    "model.layers.36.self_attn.k_proj": 63,
+    "model.layers.36.self_attn.o_proj": 78,
+    "model.layers.36.self_attn.q_proj": 63,
+    "model.layers.36.self_attn.v_proj": 77,
+    "model.layers.37.mlp.down_proj": 67,
+    "model.layers.37.mlp.gate_proj": 67,
+    "model.layers.37.mlp.up_proj": 67,
+    "model.layers.37.self_attn.k_proj": 58,
+    "model.layers.37.self_attn.o_proj": 79,
+    "model.layers.37.self_attn.q_proj": 59,
+    "model.layers.37.self_attn.v_proj": 78,
+    "model.layers.38.mlp.down_proj": 66,
+    "model.layers.38.mlp.gate_proj": 68,
+    "model.layers.38.mlp.up_proj": 67,
+    "model.layers.38.self_attn.k_proj": 57,
+    "model.layers.38.self_attn.o_proj": 81,
+    "model.layers.38.self_attn.q_proj": 57,
+    "model.layers.38.self_attn.v_proj": 81,
+    "model.layers.39.mlp.down_proj": 66,
+    "model.layers.39.mlp.gate_proj": 71,
+    "model.layers.39.mlp.up_proj": 68,
+    "model.layers.39.self_attn.k_proj": 55,
+    "model.layers.39.self_attn.o_proj": 75,
+    "model.layers.39.self_attn.q_proj": 55,
+    "model.layers.39.self_attn.v_proj": 74,
+    "model.layers.4.mlp.down_proj": 62,
+    "model.layers.4.mlp.gate_proj": 63,
+    "model.layers.4.mlp.up_proj": 62,
+    "model.layers.4.self_attn.k_proj": 66,
+    "model.layers.4.self_attn.o_proj": 52,
+    "model.layers.4.self_attn.q_proj": 66,
+    "model.layers.4.self_attn.v_proj": 52,
+    "model.layers.5.mlp.down_proj": 62,
+    "model.layers.5.mlp.gate_proj": 64,
+    "model.layers.5.mlp.up_proj": 62,
+    "model.layers.5.self_attn.k_proj": 64,
+    "model.layers.5.self_attn.o_proj": 53,
+    "model.layers.5.self_attn.q_proj": 64,
+    "model.layers.5.self_attn.v_proj": 53,
+    "model.layers.6.mlp.down_proj": 61,
+    "model.layers.6.mlp.gate_proj": 65,
+    "model.layers.6.mlp.up_proj": 62,
+    "model.layers.6.self_attn.k_proj": 69,
+    "model.layers.6.self_attn.o_proj": 55,
+    "model.layers.6.self_attn.q_proj": 68,
+    "model.layers.6.self_attn.v_proj": 55,
+    "model.layers.7.mlp.down_proj": 61,
+    "model.layers.7.mlp.gate_proj": 65,
+    "model.layers.7.mlp.up_proj": 62,
+    "model.layers.7.self_attn.k_proj": 69,
+    "model.layers.7.self_attn.o_proj": 55,
+    "model.layers.7.self_attn.q_proj": 69,
+    "model.layers.7.self_attn.v_proj": 56,
+    "model.layers.8.mlp.down_proj": 62,
+    "model.layers.8.mlp.gate_proj": 64,
+    "model.layers.8.mlp.up_proj": 63,
+    "model.layers.8.self_attn.k_proj": 68,
+    "model.layers.8.self_attn.o_proj": 56,
+    "model.layers.8.self_attn.q_proj": 67,
+    "model.layers.8.self_attn.v_proj": 56,
+    "model.layers.9.mlp.down_proj": 62,
+    "model.layers.9.mlp.gate_proj": 64,
+    "model.layers.9.mlp.up_proj": 63,
+    "model.layers.9.self_attn.k_proj": 69,
+    "model.layers.9.self_attn.o_proj": 56,
+    "model.layers.9.self_attn.q_proj": 69,
+    "model.layers.9.self_attn.v_proj": 56
+  },
+  "rank_file": "",
+  "target_modules": [
+    "gate_proj",
+    "k_proj",
+    "q_proj",
+    "down_proj",
+    "up_proj",
+    "v_proj",
+    "o_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "token_dim": 5120
+}

adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:946885901731680cd58d8c7d6b65785bd7c8b8463e7e9078f69daf3b91672099
+size 500836813

usage.py

@@ -0,0 +1,51 @@
+import sys
+sys.path.insert(1, '/workspace/asr/peft/src')
+# TODO set this path to the lazy-lora source code path, or you can install it from source code:
+# TODO, please install lazylora for usage:
+# git clone git@github.com:Xianchao-Wu/peft.git
+# cd peft
+# python setup.py install
+
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
+from peft import PeftModel, PeftConfig
+import os
+import torch
+
+import ipdb; ipdb.set_trace()
+cache_dir="/workspace/asr/peft/qlora" 
+# TODO set this cache_dir to the path where you stored (or, want to store) llama2-7bhf model
+
+lazylora_dir=os.getcwd() # the path that contains 'adapter_config.json' and 'adapter_model.bin'
+
+config = PeftConfig.from_pretrained(lazylora_dir)
+
+tokenizer = AutoTokenizer.from_pretrained(
+    config.base_model_name_or_path, 
+    cache_dir=cache_dir,
+    use_auth_token=True
+)
+
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type='nf4',
+    bnb_4bit_compute_dtype=torch.bfloat16
+)
+
+model = AutoModelForCausalLM.from_pretrained(
+    config.base_model_name_or_path,
+    quantization_config=bnb_config,
+    device_map="auto",
+    cache_dir=cache_dir,
+    use_auth_token=True
+)
+#model.print_trainable_parameters()
+print(sum(p.numel() for p in model.parameters()))
+# 6,671,979,520 -> half-size of 7B due to 4-bit loading
+
+model = PeftModel.from_pretrained(model, lazylora_dir)
+print('after adding lazy lora parameters:')
+model.print_trainable_parameters()
+# trainable params: 0 || all params: 6,922,290,688 || trainable%: 0.0
+
+