Upload 10 files

config.json

@@ -1,7 +1,7 @@
 {
   "_name_or_path": "TinyLlama/TinyLlama-1.1B-intermediate-step-955k-token-2T",
   "architectures": [
-    "LlamaForCausalLM"
+    "LlamaModel"
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
@@ -22,7 +22,7 @@
   "rope_theta": 10000.0,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.37.2",
+  "transformers_version": "4.39.3",
   "use_cache": true,
   "vocab_size": 32000
 }

loftq_init/README.md

@@ -201,4 +201,4 @@ Carbon emissions can be estimated using the [Machine Learning Impact calculator]
 
 ### Framework versions
 
-- PEFT 0.8.2
+- PEFT 0.10.0

loftq_init/adapter_config.json

@@ -1,18 +1,19 @@
 {
   "alpha_pattern": {},
   "auto_mapping": null,
-  "base_model_name_or_path": "./model_zoo/loftq/TinyLlama-1.1B-intermediate-step-955k-token-2T-2bit-32rank",
+  "base_model_name_or_path": "model/TinyLlama-1.1B-intermediate-step-955k-token-2T-4bit-32rank",
   "bias": "none",
   "fan_in_fan_out": false,
   "inference_mode": true,
   "init_lora_weights": true,
+  "layer_replication": null,
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {
-    "loftq_bits": 2,
-    "loftq_iter": 1
+    "loftq_bits": 4,
+    "loftq_iter": 5
   },
-  "lora_alpha": 32,
+  "lora_alpha": 16,
   "lora_dropout": 0.1,
   "megatron_config": null,
   "megatron_core": "megatron.core",
@@ -22,14 +23,15 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "v_proj",
-    "down_proj",
-    "k_proj",
-    "o_proj",
     "up_proj",
+    "v_proj",
     "gate_proj",
-    "q_proj"
+    "o_proj",
+    "k_proj",
+    "q_proj",
+    "down_proj"
   ],
-  "task_type": "CAUSAL_LM",
+  "task_type": "FEATURE_EXTRACTION",
+  "use_dora": false,
   "use_rslora": false
 }

loftq_init/adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6998c79c86a7f55e325290220aa34b178ff38d3c609b0539856077dcd0418a75
-size 50569626
+oid sha256:ca4828527e72c13b096312181fc2540a0746f73c8040c0f6bf80dc56ebf79895
+size 50567834

loftq_init/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:21f195928f9255a4060fd7d29ea61e5dd07751f12e81ddcd1e7a22e9352a6a85
-size 50503848
+oid sha256:c01262a21ac913f1e6039f23e4ea0d0b26f70a995617bca32641cb9041102249
+size 50502000

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9d72c89fa2c153a8e77c34b2e6bafca749e40d5b75832ff046ad061f7fd048ac
-size 2200119864
+oid sha256:46388a7344044c733d14289b9398c8e5e8f55d0c6d5f62fcc0d1f46eac4fae5d
+size 2069046544

tokenizer_config.json

@@ -35,6 +35,7 @@
   "pad_token": null,
   "padding_side": "right",
   "sp_model_kwargs": {},
+  "task_type": "feature_extraction",
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
   "use_default_system_prompt": false