KingNish commited on
Commit
1606c47
1 Parent(s): 7b4a5d0

Upload folder using huggingface_hub

Browse files
Files changed (41) hide show
  1. README.md +95 -0
  2. added_tokens.json +13 -0
  3. config.json +170 -0
  4. mergekit_config.yml +34 -0
  5. model-00001-of-00032.safetensors +3 -0
  6. model-00002-of-00032.safetensors +3 -0
  7. model-00003-of-00032.safetensors +3 -0
  8. model-00004-of-00032.safetensors +3 -0
  9. model-00005-of-00032.safetensors +3 -0
  10. model-00006-of-00032.safetensors +3 -0
  11. model-00007-of-00032.safetensors +3 -0
  12. model-00008-of-00032.safetensors +3 -0
  13. model-00009-of-00032.safetensors +3 -0
  14. model-00010-of-00032.safetensors +3 -0
  15. model-00011-of-00032.safetensors +3 -0
  16. model-00012-of-00032.safetensors +3 -0
  17. model-00013-of-00032.safetensors +3 -0
  18. model-00014-of-00032.safetensors +3 -0
  19. model-00015-of-00032.safetensors +3 -0
  20. model-00016-of-00032.safetensors +3 -0
  21. model-00017-of-00032.safetensors +3 -0
  22. model-00018-of-00032.safetensors +3 -0
  23. model-00019-of-00032.safetensors +3 -0
  24. model-00020-of-00032.safetensors +3 -0
  25. model-00021-of-00032.safetensors +3 -0
  26. model-00022-of-00032.safetensors +3 -0
  27. model-00023-of-00032.safetensors +3 -0
  28. model-00024-of-00032.safetensors +3 -0
  29. model-00025-of-00032.safetensors +3 -0
  30. model-00026-of-00032.safetensors +3 -0
  31. model-00027-of-00032.safetensors +3 -0
  32. model-00028-of-00032.safetensors +3 -0
  33. model-00029-of-00032.safetensors +3 -0
  34. model-00030-of-00032.safetensors +3 -0
  35. model-00031-of-00032.safetensors +3 -0
  36. model-00032-of-00032.safetensors +3 -0
  37. model.safetensors.index.json +1 -0
  38. special_tokens_map.json +30 -0
  39. tokenizer.json +0 -0
  40. tokenizer.model +3 -0
  41. tokenizer_config.json +131 -0
README.md ADDED
@@ -0,0 +1,95 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model:
3
+ - microsoft/Phi-3-medium-128k-instruct
4
+ - microsoft/Phi-3-medium-128k-instruct
5
+ - microsoft/Phi-3-medium-128k-instruct
6
+ - microsoft/Phi-3-medium-128k-instruct
7
+ - microsoft/Phi-3-medium-128k-instruct
8
+ - microsoft/Phi-3-medium-128k-instruct
9
+ - microsoft/Phi-3-medium-128k-instruct
10
+ - microsoft/Phi-3-medium-128k-instruct
11
+ - microsoft/Phi-3-medium-128k-instruct
12
+ - microsoft/Phi-3-medium-128k-instruct
13
+ tags:
14
+ - merge
15
+ - mergekit
16
+ - lazymergekit
17
+ - microsoft/Phi-3-medium-128k-instruct
18
+ ---
19
+
20
+ # experiment-13
21
+
22
+ experiment-13 is a merge of the following models using [LazyMergekit](https://colab.research.google.com/drive/1obulZ1ROXHjYLn6PPZJwRR6GzgQogxxb?usp=sharing):
23
+ * [microsoft/Phi-3-medium-128k-instruct](https://huggingface.co/microsoft/Phi-3-medium-128k-instruct)
24
+ * [microsoft/Phi-3-medium-128k-instruct](https://huggingface.co/microsoft/Phi-3-medium-128k-instruct)
25
+ * [microsoft/Phi-3-medium-128k-instruct](https://huggingface.co/microsoft/Phi-3-medium-128k-instruct)
26
+ * [microsoft/Phi-3-medium-128k-instruct](https://huggingface.co/microsoft/Phi-3-medium-128k-instruct)
27
+ * [microsoft/Phi-3-medium-128k-instruct](https://huggingface.co/microsoft/Phi-3-medium-128k-instruct)
28
+ * [microsoft/Phi-3-medium-128k-instruct](https://huggingface.co/microsoft/Phi-3-medium-128k-instruct)
29
+ * [microsoft/Phi-3-medium-128k-instruct](https://huggingface.co/microsoft/Phi-3-medium-128k-instruct)
30
+ * [microsoft/Phi-3-medium-128k-instruct](https://huggingface.co/microsoft/Phi-3-medium-128k-instruct)
31
+ * [microsoft/Phi-3-medium-128k-instruct](https://huggingface.co/microsoft/Phi-3-medium-128k-instruct)
32
+ * [microsoft/Phi-3-medium-128k-instruct](https://huggingface.co/microsoft/Phi-3-medium-128k-instruct)
33
+
34
+ ## 🧩 Configuration
35
+
36
+ ```yaml
37
+ slices:
38
+ - sources:
39
+ - layer_range: [0, 4]
40
+ model: microsoft/Phi-3-medium-128k-instruct
41
+ - sources:
42
+ - layer_range: [5, 8]
43
+ model: microsoft/Phi-3-medium-128k-instruct
44
+ - sources:
45
+ - layer_range: [9, 12]
46
+ model: microsoft/Phi-3-medium-128k-instruct
47
+ - sources:
48
+ - layer_range: [13, 16]
49
+ model: microsoft/Phi-3-medium-128k-instruct
50
+ - sources:
51
+ - layer_range: [17, 20]
52
+ model: microsoft/Phi-3-medium-128k-instruct
53
+ - sources:
54
+ - layer_range: [21, 24]
55
+ model: microsoft/Phi-3-medium-128k-instruct
56
+ - sources:
57
+ - layer_range: [25, 28]
58
+ model: microsoft/Phi-3-medium-128k-instruct
59
+ - sources:
60
+ - layer_range: [29, 32]
61
+ model: microsoft/Phi-3-medium-128k-instruct
62
+ - sources:
63
+ - layer_range: [33, 36]
64
+ model: microsoft/Phi-3-medium-128k-instruct
65
+ - sources:
66
+ - layer_range: [37, 40]
67
+ model: microsoft/Phi-3-medium-128k-instruct
68
+ merge_method: passthrough
69
+ dtype: bfloat16
70
+ ```
71
+
72
+ ## 💻 Usage
73
+
74
+ ```python
75
+ !pip install -qU transformers accelerate
76
+
77
+ from transformers import AutoTokenizer
78
+ import transformers
79
+ import torch
80
+
81
+ model = "KingNish/experiment-13"
82
+ messages = [{"role": "user", "content": "What is a large language model?"}]
83
+
84
+ tokenizer = AutoTokenizer.from_pretrained(model)
85
+ prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
86
+ pipeline = transformers.pipeline(
87
+ "text-generation",
88
+ model=model,
89
+ torch_dtype=torch.float16,
90
+ device_map="auto",
91
+ )
92
+
93
+ outputs = pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)
94
+ print(outputs[0]["generated_text"])
95
+ ```
added_tokens.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "<|assistant|>": 32001,
3
+ "<|endoftext|>": 32000,
4
+ "<|end|>": 32007,
5
+ "<|placeholder1|>": 32002,
6
+ "<|placeholder2|>": 32003,
7
+ "<|placeholder3|>": 32004,
8
+ "<|placeholder4|>": 32005,
9
+ "<|placeholder5|>": 32008,
10
+ "<|placeholder6|>": 32009,
11
+ "<|system|>": 32006,
12
+ "<|user|>": 32010
13
+ }
config.json ADDED
@@ -0,0 +1,170 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "microsoft/Phi-3-medium-128k-instruct",
3
+ "architectures": [
4
+ "Phi3ForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "auto_map": {
9
+ "AutoConfig": "microsoft/Phi-3-medium-128k-instruct--configuration_phi3.Phi3Config",
10
+ "AutoModelForCausalLM": "microsoft/Phi-3-medium-128k-instruct--modeling_phi3.Phi3ForCausalLM"
11
+ },
12
+ "bos_token_id": 1,
13
+ "embd_pdrop": 0.0,
14
+ "eos_token_id": 32000,
15
+ "hidden_act": "silu",
16
+ "hidden_size": 5120,
17
+ "initializer_range": 0.02,
18
+ "intermediate_size": 17920,
19
+ "max_position_embeddings": 131072,
20
+ "model_type": "phi3",
21
+ "num_attention_heads": 40,
22
+ "num_hidden_layers": 31,
23
+ "num_key_value_heads": 10,
24
+ "original_max_position_embeddings": 4096,
25
+ "pad_token_id": null,
26
+ "resid_pdrop": 0.0,
27
+ "rms_norm_eps": 1e-05,
28
+ "rope_scaling": {
29
+ "long_factor": [
30
+ 1.0,
31
+ 1.0,
32
+ 1.0,
33
+ 1.0,
34
+ 1.0,
35
+ 1.0,
36
+ 1.0,
37
+ 1.0,
38
+ 1.0,
39
+ 1.0,
40
+ 1.0,
41
+ 1.0,
42
+ 1.0,
43
+ 1.25,
44
+ 1.25,
45
+ 1.5,
46
+ 2.0,
47
+ 2.75,
48
+ 5.75,
49
+ 5.75,
50
+ 6.5,
51
+ 9.25,
52
+ 11.0,
53
+ 13.25,
54
+ 19.25,
55
+ 19.75,
56
+ 19.75,
57
+ 21.25,
58
+ 21.5,
59
+ 26.5,
60
+ 30.0,
61
+ 33.75,
62
+ 35.25,
63
+ 38.5,
64
+ 42.0,
65
+ 42.25,
66
+ 46.0,
67
+ 47.0,
68
+ 50.0,
69
+ 50.5,
70
+ 51.0,
71
+ 52.0,
72
+ 52.75,
73
+ 53.75,
74
+ 54.75,
75
+ 57.0,
76
+ 57.25,
77
+ 58.5,
78
+ 59.25,
79
+ 59.5,
80
+ 62.0,
81
+ 62.5,
82
+ 62.75,
83
+ 63.25,
84
+ 63.25,
85
+ 63.25,
86
+ 63.75,
87
+ 64.0,
88
+ 64.0,
89
+ 64.25,
90
+ 64.5,
91
+ 64.5,
92
+ 65.0,
93
+ 65.0
94
+ ],
95
+ "short_factor": [
96
+ 1.0,
97
+ 1.0,
98
+ 1.0,
99
+ 1.0,
100
+ 1.0,
101
+ 1.0,
102
+ 1.01,
103
+ 1.02,
104
+ 1.02,
105
+ 1.04,
106
+ 1.04,
107
+ 1.07,
108
+ 1.07,
109
+ 1.1,
110
+ 1.3000000000000003,
111
+ 1.3000000000000003,
112
+ 1.5000000000000004,
113
+ 1.5700000000000005,
114
+ 1.9000000000000008,
115
+ 2.3100000000000014,
116
+ 2.759999999999992,
117
+ 3.3899999999999784,
118
+ 3.9399999999999666,
119
+ 4.009999999999965,
120
+ 4.289999999999959,
121
+ 4.349999999999958,
122
+ 5.349999999999937,
123
+ 6.659999999999909,
124
+ 7.029999999999901,
125
+ 7.51999999999989,
126
+ 8.00999999999988,
127
+ 8.249999999999876,
128
+ 8.279999999999875,
129
+ 9.629999999999846,
130
+ 9.89999999999984,
131
+ 10.589999999999826,
132
+ 11.049999999999816,
133
+ 11.7899999999998,
134
+ 12.189999999999792,
135
+ 12.889999999999777,
136
+ 13.129999999999772,
137
+ 13.16999999999977,
138
+ 13.20999999999977,
139
+ 13.479999999999764,
140
+ 13.539999999999763,
141
+ 13.779999999999758,
142
+ 13.929999999999755,
143
+ 14.429999999999744,
144
+ 14.759999999999737,
145
+ 15.149999999999729,
146
+ 15.419999999999723,
147
+ 15.53999999999972,
148
+ 15.659999999999718,
149
+ 15.749999999999716,
150
+ 15.759999999999716,
151
+ 15.799999999999715,
152
+ 16.05999999999971,
153
+ 16.079999999999714,
154
+ 16.11999999999972,
155
+ 16.11999999999972,
156
+ 16.18999999999973,
157
+ 16.31999999999975,
158
+ 16.539999999999786,
159
+ 16.799999999999827
160
+ ],
161
+ "type": "su"
162
+ },
163
+ "rope_theta": 10000.0,
164
+ "sliding_window": 131072,
165
+ "tie_word_embeddings": false,
166
+ "torch_dtype": "bfloat16",
167
+ "transformers_version": "4.42.4",
168
+ "use_cache": true,
169
+ "vocab_size": 32064
170
+ }
mergekit_config.yml ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ slices:
3
+ - sources:
4
+ - layer_range: [0, 4]
5
+ model: microsoft/Phi-3-medium-128k-instruct
6
+ - sources:
7
+ - layer_range: [5, 8]
8
+ model: microsoft/Phi-3-medium-128k-instruct
9
+ - sources:
10
+ - layer_range: [9, 12]
11
+ model: microsoft/Phi-3-medium-128k-instruct
12
+ - sources:
13
+ - layer_range: [13, 16]
14
+ model: microsoft/Phi-3-medium-128k-instruct
15
+ - sources:
16
+ - layer_range: [17, 20]
17
+ model: microsoft/Phi-3-medium-128k-instruct
18
+ - sources:
19
+ - layer_range: [21, 24]
20
+ model: microsoft/Phi-3-medium-128k-instruct
21
+ - sources:
22
+ - layer_range: [25, 28]
23
+ model: microsoft/Phi-3-medium-128k-instruct
24
+ - sources:
25
+ - layer_range: [29, 32]
26
+ model: microsoft/Phi-3-medium-128k-instruct
27
+ - sources:
28
+ - layer_range: [33, 36]
29
+ model: microsoft/Phi-3-medium-128k-instruct
30
+ - sources:
31
+ - layer_range: [37, 40]
32
+ model: microsoft/Phi-3-medium-128k-instruct
33
+ merge_method: passthrough
34
+ dtype: bfloat16
model-00001-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cd319676683bc4a74924af0037f325b4bf9a313717cbfa30bf9b6b7cf69aee19
3
+ size 840182216
model-00002-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7cd490d39eb5f9554d2d91cef03b3223b5a1b705d0237c5bc127fff02d5d46ad
3
+ size 681595600
model-00003-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:151e672eb67fc415885357c6d4dc1b1ceb86aab76cb6a613a9bf900aad894c18
3
+ size 681595600
model-00004-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b2484c2c07276a0f41ac8c6b9765da108df6e7ae27dad6ac89efa6a2b5590f2d
3
+ size 681595600
model-00005-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0b684ed782f0e47eecbb16b96996ec3bab019a904f24ea00ead8b4840d133cc9
3
+ size 681595600
model-00006-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:386d6cb5c81afe269b0c0013eb2848f790f5fb667b2e2c9102ba698a39c8e474
3
+ size 681595608
model-00007-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:850f61d02b395bac15e537c3eb085d2719ece1ebdee415e7e7abbe307e228dc4
3
+ size 681595608
model-00008-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:caa298523147c4260d79d791a0261a8c902dabc17d53f26585d7978a400cb095
3
+ size 681595608
model-00009-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:20a01ce07beab6726ba210ce142b8dee4c863b318bbe53768f3432a4c0daff14
3
+ size 681595608
model-00010-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5a33cfa93b23893047ba4c9962c1f949c2cc7cdcb19562b75bc6b751ce3b9b80
3
+ size 681595608
model-00011-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:136a39745f175fa746a1d31da04701ed88579b546f69c0711eca6d519c7d7f09
3
+ size 681595608
model-00012-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c4f0f1a587f0110447c0d5f5d4e8e85b97470152791af9f6a509254800bad717
3
+ size 681595600
model-00013-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:48da3783e481692773a09426cf505b412fbd8fad402bf947f18128f878758ea2
3
+ size 681595608
model-00014-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2d219c5be8c978703ee0ca6c44abc1cbbee7f406a80c908a516405437f4c70ef
3
+ size 681595608
model-00015-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7acc70815f3313b3c8964316078a696990feac0c2ac8329145581e7f78d097e7
3
+ size 681595608
model-00016-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e39811cfc2e713b576014c083ff9be15a1afec8a5f6f3fa8302877a72c5b06cc
3
+ size 681595608
model-00017-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0e571711a1a8f4f1c833984692e411afcbb81c43dfc49725dba8a4c21386d59c
3
+ size 681595608
model-00018-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9639b8863dfd88c6f43da714cff3e020807210c73dcc1c0403c7107f0679fc3b
3
+ size 681595608
model-00019-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:afc6e235ed9c47e7a6f0df6c7ed9e79a934a8e94e8e7924da9aa7e874c31283e
3
+ size 681595608
model-00020-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0a8291e08085c0a39491fc4be5fb3583c724093abbf74c856bfe3ab4ff9435fb
3
+ size 681595600
model-00021-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5286ae17e11699cd2fe060ebbf23747e3f6ea24f16b1b9b23a5d21f3dda7c209
3
+ size 681595608
model-00022-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a37539ed55b8e960bea7db4424bb7881df4d3c90cca8ac5f5eacce37c8489fa7
3
+ size 681595608
model-00023-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:426e6e7c92a90f06367c118742d098f9bc4871a21eff24569ac1a5b8e2a7a4a1
3
+ size 681595608
model-00024-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6e61c85c4b7d7d41d60184510786e60f3004a3615cfef8a647f6e7ff47ffcff7
3
+ size 681595608
model-00025-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:750c1b28b948a2109a53c4cbce7053a4cdeb13956bd17bd0e42074539664d5ac
3
+ size 681595608
model-00026-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0525907323f2159c01c5880819d5def306a360bb80490564c82501963a3ef548
3
+ size 681595608
model-00027-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e7347d8ba5889a6371165dfb03a7d11e598bbfb27314941d3a6792c40344e593
3
+ size 681595608
model-00028-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a74f7e9d0e7f0b51aef56bd7198c2ac1d16a9672b65ecff38be9c723379a9d7f
3
+ size 681595608
model-00029-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0b5c5612141e1ac1f66b6baea5398d7bbc3ef3c5ac76fcc43f383de1c4f85d97
3
+ size 681595600
model-00030-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cb1b9945676e4d5420bda0151f8cdd14c9af436422baaa17b196de2c25d0ae92
3
+ size 681595600
model-00031-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:847fd1ed7c3dbd846d8525a6dd8591307b89c53d11f1221ce92119057c2f0853
3
+ size 681595600
model-00032-of-00032.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2416250a366e93961e4f1f1333847557564c02df2dd81d9ce947181af997f319
3
+ size 498094672
model.safetensors.index.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"metadata": {"mergekit_version": "0.0.4.4", "total_size": 21786122240}, "weight_map": {"lm_head.weight": "model-00001-of-00032.safetensors", "model.embed_tokens.weight": "model-00001-of-00032.safetensors", "model.layers.0.input_layernorm.weight": "model-00001-of-00032.safetensors", "model.layers.0.mlp.down_proj.weight": "model-00001-of-00032.safetensors", "model.layers.0.mlp.gate_up_proj.weight": "model-00002-of-00032.safetensors", "model.layers.0.post_attention_layernorm.weight": "model-00002-of-00032.safetensors", "model.layers.0.self_attn.o_proj.weight": "model-00002-of-00032.safetensors", "model.layers.0.self_attn.qkv_proj.weight": "model-00002-of-00032.safetensors", "model.layers.1.input_layernorm.weight": "model-00002-of-00032.safetensors", "model.layers.1.mlp.down_proj.weight": "model-00002-of-00032.safetensors", "model.layers.1.mlp.gate_up_proj.weight": "model-00003-of-00032.safetensors", "model.layers.1.post_attention_layernorm.weight": "model-00003-of-00032.safetensors", "model.layers.1.self_attn.o_proj.weight": "model-00003-of-00032.safetensors", "model.layers.1.self_attn.qkv_proj.weight": "model-00003-of-00032.safetensors", "model.layers.8.input_layernorm.weight": "model-00003-of-00032.safetensors", "model.layers.8.mlp.down_proj.weight": "model-00003-of-00032.safetensors", "model.layers.8.mlp.gate_up_proj.weight": "model-00004-of-00032.safetensors", "model.layers.8.post_attention_layernorm.weight": "model-00004-of-00032.safetensors", "model.layers.8.self_attn.o_proj.weight": "model-00004-of-00032.safetensors", "model.layers.8.self_attn.qkv_proj.weight": "model-00004-of-00032.safetensors", "model.layers.9.input_layernorm.weight": "model-00004-of-00032.safetensors", "model.layers.9.mlp.down_proj.weight": "model-00004-of-00032.safetensors", "model.layers.9.mlp.gate_up_proj.weight": "model-00005-of-00032.safetensors", "model.layers.9.post_attention_layernorm.weight": "model-00005-of-00032.safetensors", "model.layers.9.self_attn.o_proj.weight": "model-00005-of-00032.safetensors", "model.layers.9.self_attn.qkv_proj.weight": "model-00005-of-00032.safetensors", "model.layers.10.input_layernorm.weight": "model-00005-of-00032.safetensors", "model.layers.10.mlp.down_proj.weight": "model-00005-of-00032.safetensors", "model.layers.10.mlp.gate_up_proj.weight": "model-00006-of-00032.safetensors", "model.layers.10.post_attention_layernorm.weight": "model-00006-of-00032.safetensors", "model.layers.10.self_attn.o_proj.weight": "model-00006-of-00032.safetensors", "model.layers.10.self_attn.qkv_proj.weight": "model-00006-of-00032.safetensors", "model.layers.11.input_layernorm.weight": "model-00006-of-00032.safetensors", "model.layers.11.mlp.down_proj.weight": "model-00006-of-00032.safetensors", "model.layers.11.mlp.gate_up_proj.weight": "model-00007-of-00032.safetensors", "model.layers.11.post_attention_layernorm.weight": "model-00007-of-00032.safetensors", "model.layers.11.self_attn.o_proj.weight": "model-00007-of-00032.safetensors", "model.layers.11.self_attn.qkv_proj.weight": "model-00007-of-00032.safetensors", "model.layers.12.input_layernorm.weight": "model-00007-of-00032.safetensors", "model.layers.12.mlp.down_proj.weight": "model-00007-of-00032.safetensors", "model.layers.12.mlp.gate_up_proj.weight": "model-00008-of-00032.safetensors", "model.layers.12.post_attention_layernorm.weight": "model-00008-of-00032.safetensors", "model.layers.12.self_attn.o_proj.weight": "model-00008-of-00032.safetensors", "model.layers.12.self_attn.qkv_proj.weight": "model-00008-of-00032.safetensors", "model.layers.13.input_layernorm.weight": "model-00008-of-00032.safetensors", "model.layers.13.mlp.down_proj.weight": "model-00008-of-00032.safetensors", "model.layers.13.mlp.gate_up_proj.weight": "model-00009-of-00032.safetensors", "model.layers.13.post_attention_layernorm.weight": "model-00009-of-00032.safetensors", "model.layers.13.self_attn.o_proj.weight": "model-00009-of-00032.safetensors", "model.layers.13.self_attn.qkv_proj.weight": "model-00009-of-00032.safetensors", "model.layers.14.input_layernorm.weight": "model-00009-of-00032.safetensors", "model.layers.14.mlp.down_proj.weight": "model-00009-of-00032.safetensors", "model.layers.14.mlp.gate_up_proj.weight": "model-00010-of-00032.safetensors", "model.layers.14.post_attention_layernorm.weight": "model-00010-of-00032.safetensors", "model.layers.14.self_attn.o_proj.weight": "model-00010-of-00032.safetensors", "model.layers.14.self_attn.qkv_proj.weight": "model-00010-of-00032.safetensors", "model.layers.15.input_layernorm.weight": "model-00010-of-00032.safetensors", "model.layers.15.mlp.down_proj.weight": "model-00010-of-00032.safetensors", "model.layers.15.mlp.gate_up_proj.weight": "model-00011-of-00032.safetensors", "model.layers.15.post_attention_layernorm.weight": "model-00011-of-00032.safetensors", "model.layers.15.self_attn.o_proj.weight": "model-00011-of-00032.safetensors", "model.layers.15.self_attn.qkv_proj.weight": "model-00011-of-00032.safetensors", "model.layers.2.input_layernorm.weight": "model-00011-of-00032.safetensors", "model.layers.2.mlp.down_proj.weight": "model-00011-of-00032.safetensors", "model.layers.2.mlp.gate_up_proj.weight": "model-00012-of-00032.safetensors", "model.layers.2.post_attention_layernorm.weight": "model-00012-of-00032.safetensors", "model.layers.2.self_attn.o_proj.weight": "model-00012-of-00032.safetensors", "model.layers.2.self_attn.qkv_proj.weight": "model-00012-of-00032.safetensors", "model.layers.16.input_layernorm.weight": "model-00012-of-00032.safetensors", "model.layers.16.mlp.down_proj.weight": "model-00012-of-00032.safetensors", "model.layers.16.mlp.gate_up_proj.weight": "model-00013-of-00032.safetensors", "model.layers.16.post_attention_layernorm.weight": "model-00013-of-00032.safetensors", "model.layers.16.self_attn.o_proj.weight": "model-00013-of-00032.safetensors", "model.layers.16.self_attn.qkv_proj.weight": "model-00013-of-00032.safetensors", "model.layers.17.input_layernorm.weight": "model-00013-of-00032.safetensors", "model.layers.17.mlp.down_proj.weight": "model-00013-of-00032.safetensors", "model.layers.17.mlp.gate_up_proj.weight": "model-00014-of-00032.safetensors", "model.layers.17.post_attention_layernorm.weight": "model-00014-of-00032.safetensors", "model.layers.17.self_attn.o_proj.weight": "model-00014-of-00032.safetensors", "model.layers.17.self_attn.qkv_proj.weight": "model-00014-of-00032.safetensors", "model.layers.18.input_layernorm.weight": "model-00014-of-00032.safetensors", "model.layers.18.mlp.down_proj.weight": "model-00014-of-00032.safetensors", "model.layers.18.mlp.gate_up_proj.weight": "model-00015-of-00032.safetensors", "model.layers.18.post_attention_layernorm.weight": "model-00015-of-00032.safetensors", "model.layers.18.self_attn.o_proj.weight": "model-00015-of-00032.safetensors", "model.layers.18.self_attn.qkv_proj.weight": "model-00015-of-00032.safetensors", "model.layers.19.input_layernorm.weight": "model-00015-of-00032.safetensors", "model.layers.19.mlp.down_proj.weight": "model-00015-of-00032.safetensors", "model.layers.19.mlp.gate_up_proj.weight": "model-00016-of-00032.safetensors", "model.layers.19.post_attention_layernorm.weight": "model-00016-of-00032.safetensors", "model.layers.19.self_attn.o_proj.weight": "model-00016-of-00032.safetensors", "model.layers.19.self_attn.qkv_proj.weight": "model-00016-of-00032.safetensors", "model.layers.20.input_layernorm.weight": "model-00016-of-00032.safetensors", "model.layers.20.mlp.down_proj.weight": "model-00016-of-00032.safetensors", "model.layers.20.mlp.gate_up_proj.weight": "model-00017-of-00032.safetensors", "model.layers.20.post_attention_layernorm.weight": "model-00017-of-00032.safetensors", "model.layers.20.self_attn.o_proj.weight": "model-00017-of-00032.safetensors", "model.layers.20.self_attn.qkv_proj.weight": "model-00017-of-00032.safetensors", "model.layers.21.input_layernorm.weight": "model-00017-of-00032.safetensors", "model.layers.21.mlp.down_proj.weight": "model-00017-of-00032.safetensors", "model.layers.21.mlp.gate_up_proj.weight": "model-00018-of-00032.safetensors", "model.layers.21.post_attention_layernorm.weight": "model-00018-of-00032.safetensors", "model.layers.21.self_attn.o_proj.weight": "model-00018-of-00032.safetensors", "model.layers.21.self_attn.qkv_proj.weight": "model-00018-of-00032.safetensors", "model.layers.22.input_layernorm.weight": "model-00018-of-00032.safetensors", "model.layers.22.mlp.down_proj.weight": "model-00018-of-00032.safetensors", "model.layers.22.mlp.gate_up_proj.weight": "model-00019-of-00032.safetensors", "model.layers.22.post_attention_layernorm.weight": "model-00019-of-00032.safetensors", "model.layers.22.self_attn.o_proj.weight": "model-00019-of-00032.safetensors", "model.layers.22.self_attn.qkv_proj.weight": "model-00019-of-00032.safetensors", "model.layers.3.input_layernorm.weight": "model-00019-of-00032.safetensors", "model.layers.3.mlp.down_proj.weight": "model-00019-of-00032.safetensors", "model.layers.3.mlp.gate_up_proj.weight": "model-00020-of-00032.safetensors", "model.layers.3.post_attention_layernorm.weight": "model-00020-of-00032.safetensors", "model.layers.3.self_attn.o_proj.weight": "model-00020-of-00032.safetensors", "model.layers.3.self_attn.qkv_proj.weight": "model-00020-of-00032.safetensors", "model.layers.23.input_layernorm.weight": "model-00020-of-00032.safetensors", "model.layers.23.mlp.down_proj.weight": "model-00020-of-00032.safetensors", "model.layers.23.mlp.gate_up_proj.weight": "model-00021-of-00032.safetensors", "model.layers.23.post_attention_layernorm.weight": "model-00021-of-00032.safetensors", "model.layers.23.self_attn.o_proj.weight": "model-00021-of-00032.safetensors", "model.layers.23.self_attn.qkv_proj.weight": "model-00021-of-00032.safetensors", "model.layers.24.input_layernorm.weight": "model-00021-of-00032.safetensors", "model.layers.24.mlp.down_proj.weight": "model-00021-of-00032.safetensors", "model.layers.24.mlp.gate_up_proj.weight": "model-00022-of-00032.safetensors", "model.layers.24.post_attention_layernorm.weight": "model-00022-of-00032.safetensors", "model.layers.24.self_attn.o_proj.weight": "model-00022-of-00032.safetensors", "model.layers.24.self_attn.qkv_proj.weight": "model-00022-of-00032.safetensors", "model.layers.25.input_layernorm.weight": "model-00022-of-00032.safetensors", "model.layers.25.mlp.down_proj.weight": "model-00022-of-00032.safetensors", "model.layers.25.mlp.gate_up_proj.weight": "model-00023-of-00032.safetensors", "model.layers.25.post_attention_layernorm.weight": "model-00023-of-00032.safetensors", "model.layers.25.self_attn.o_proj.weight": "model-00023-of-00032.safetensors", "model.layers.25.self_attn.qkv_proj.weight": "model-00023-of-00032.safetensors", "model.layers.26.input_layernorm.weight": "model-00023-of-00032.safetensors", "model.layers.26.mlp.down_proj.weight": "model-00023-of-00032.safetensors", "model.layers.26.mlp.gate_up_proj.weight": "model-00024-of-00032.safetensors", "model.layers.26.post_attention_layernorm.weight": "model-00024-of-00032.safetensors", "model.layers.26.self_attn.o_proj.weight": "model-00024-of-00032.safetensors", "model.layers.26.self_attn.qkv_proj.weight": "model-00024-of-00032.safetensors", "model.layers.27.input_layernorm.weight": "model-00024-of-00032.safetensors", "model.layers.27.mlp.down_proj.weight": "model-00024-of-00032.safetensors", "model.layers.27.mlp.gate_up_proj.weight": "model-00025-of-00032.safetensors", "model.layers.27.post_attention_layernorm.weight": "model-00025-of-00032.safetensors", "model.layers.27.self_attn.o_proj.weight": "model-00025-of-00032.safetensors", "model.layers.27.self_attn.qkv_proj.weight": "model-00025-of-00032.safetensors", "model.layers.28.input_layernorm.weight": "model-00025-of-00032.safetensors", "model.layers.28.mlp.down_proj.weight": "model-00025-of-00032.safetensors", "model.layers.28.mlp.gate_up_proj.weight": "model-00026-of-00032.safetensors", "model.layers.28.post_attention_layernorm.weight": "model-00026-of-00032.safetensors", "model.layers.28.self_attn.o_proj.weight": "model-00026-of-00032.safetensors", "model.layers.28.self_attn.qkv_proj.weight": "model-00026-of-00032.safetensors", "model.layers.29.input_layernorm.weight": "model-00026-of-00032.safetensors", "model.layers.29.mlp.down_proj.weight": "model-00026-of-00032.safetensors", "model.layers.29.mlp.gate_up_proj.weight": "model-00027-of-00032.safetensors", "model.layers.29.post_attention_layernorm.weight": "model-00027-of-00032.safetensors", "model.layers.29.self_attn.o_proj.weight": "model-00027-of-00032.safetensors", "model.layers.29.self_attn.qkv_proj.weight": "model-00027-of-00032.safetensors", "model.layers.30.input_layernorm.weight": "model-00027-of-00032.safetensors", "model.layers.30.mlp.down_proj.weight": "model-00027-of-00032.safetensors", "model.layers.30.mlp.gate_up_proj.weight": "model-00028-of-00032.safetensors", "model.layers.30.post_attention_layernorm.weight": "model-00028-of-00032.safetensors", "model.layers.30.self_attn.o_proj.weight": "model-00028-of-00032.safetensors", "model.layers.30.self_attn.qkv_proj.weight": "model-00028-of-00032.safetensors", "model.layers.4.input_layernorm.weight": "model-00028-of-00032.safetensors", "model.layers.4.mlp.down_proj.weight": "model-00028-of-00032.safetensors", "model.layers.4.mlp.gate_up_proj.weight": "model-00029-of-00032.safetensors", "model.layers.4.post_attention_layernorm.weight": "model-00029-of-00032.safetensors", "model.layers.4.self_attn.o_proj.weight": "model-00029-of-00032.safetensors", "model.layers.4.self_attn.qkv_proj.weight": "model-00029-of-00032.safetensors", "model.layers.5.input_layernorm.weight": "model-00029-of-00032.safetensors", "model.layers.5.mlp.down_proj.weight": "model-00029-of-00032.safetensors", "model.layers.5.mlp.gate_up_proj.weight": "model-00030-of-00032.safetensors", "model.layers.5.post_attention_layernorm.weight": "model-00030-of-00032.safetensors", "model.layers.5.self_attn.o_proj.weight": "model-00030-of-00032.safetensors", "model.layers.5.self_attn.qkv_proj.weight": "model-00030-of-00032.safetensors", "model.layers.6.input_layernorm.weight": "model-00030-of-00032.safetensors", "model.layers.6.mlp.down_proj.weight": "model-00030-of-00032.safetensors", "model.layers.6.mlp.gate_up_proj.weight": "model-00031-of-00032.safetensors", "model.layers.6.post_attention_layernorm.weight": "model-00031-of-00032.safetensors", "model.layers.6.self_attn.o_proj.weight": "model-00031-of-00032.safetensors", "model.layers.6.self_attn.qkv_proj.weight": "model-00031-of-00032.safetensors", "model.layers.7.input_layernorm.weight": "model-00031-of-00032.safetensors", "model.layers.7.mlp.down_proj.weight": "model-00031-of-00032.safetensors", "model.layers.7.mlp.gate_up_proj.weight": "model-00032-of-00032.safetensors", "model.layers.7.post_attention_layernorm.weight": "model-00032-of-00032.safetensors", "model.layers.7.self_attn.o_proj.weight": "model-00032-of-00032.safetensors", "model.layers.7.self_attn.qkv_proj.weight": "model-00032-of-00032.safetensors", "model.norm.weight": "model-00032-of-00032.safetensors"}}
special_tokens_map.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "<|endoftext|>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": {
17
+ "content": "<|endoftext|>",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "unk_token": {
24
+ "content": "<unk>",
25
+ "lstrip": false,
26
+ "normalized": false,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ }
30
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
3
+ size 499723
tokenizer_config.json ADDED
@@ -0,0 +1,131 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_eos_token": false,
4
+ "add_prefix_space": null,
5
+ "added_tokens_decoder": {
6
+ "0": {
7
+ "content": "<unk>",
8
+ "lstrip": false,
9
+ "normalized": false,
10
+ "rstrip": false,
11
+ "single_word": false,
12
+ "special": true
13
+ },
14
+ "1": {
15
+ "content": "<s>",
16
+ "lstrip": false,
17
+ "normalized": false,
18
+ "rstrip": false,
19
+ "single_word": false,
20
+ "special": true
21
+ },
22
+ "2": {
23
+ "content": "</s>",
24
+ "lstrip": false,
25
+ "normalized": false,
26
+ "rstrip": true,
27
+ "single_word": false,
28
+ "special": false
29
+ },
30
+ "32000": {
31
+ "content": "<|endoftext|>",
32
+ "lstrip": false,
33
+ "normalized": false,
34
+ "rstrip": false,
35
+ "single_word": false,
36
+ "special": true
37
+ },
38
+ "32001": {
39
+ "content": "<|assistant|>",
40
+ "lstrip": false,
41
+ "normalized": false,
42
+ "rstrip": true,
43
+ "single_word": false,
44
+ "special": true
45
+ },
46
+ "32002": {
47
+ "content": "<|placeholder1|>",
48
+ "lstrip": false,
49
+ "normalized": false,
50
+ "rstrip": true,
51
+ "single_word": false,
52
+ "special": true
53
+ },
54
+ "32003": {
55
+ "content": "<|placeholder2|>",
56
+ "lstrip": false,
57
+ "normalized": false,
58
+ "rstrip": true,
59
+ "single_word": false,
60
+ "special": true
61
+ },
62
+ "32004": {
63
+ "content": "<|placeholder3|>",
64
+ "lstrip": false,
65
+ "normalized": false,
66
+ "rstrip": true,
67
+ "single_word": false,
68
+ "special": true
69
+ },
70
+ "32005": {
71
+ "content": "<|placeholder4|>",
72
+ "lstrip": false,
73
+ "normalized": false,
74
+ "rstrip": true,
75
+ "single_word": false,
76
+ "special": true
77
+ },
78
+ "32006": {
79
+ "content": "<|system|>",
80
+ "lstrip": false,
81
+ "normalized": false,
82
+ "rstrip": true,
83
+ "single_word": false,
84
+ "special": true
85
+ },
86
+ "32007": {
87
+ "content": "<|end|>",
88
+ "lstrip": false,
89
+ "normalized": false,
90
+ "rstrip": true,
91
+ "single_word": false,
92
+ "special": true
93
+ },
94
+ "32008": {
95
+ "content": "<|placeholder5|>",
96
+ "lstrip": false,
97
+ "normalized": false,
98
+ "rstrip": true,
99
+ "single_word": false,
100
+ "special": true
101
+ },
102
+ "32009": {
103
+ "content": "<|placeholder6|>",
104
+ "lstrip": false,
105
+ "normalized": false,
106
+ "rstrip": true,
107
+ "single_word": false,
108
+ "special": true
109
+ },
110
+ "32010": {
111
+ "content": "<|user|>",
112
+ "lstrip": false,
113
+ "normalized": false,
114
+ "rstrip": true,
115
+ "single_word": false,
116
+ "special": true
117
+ }
118
+ },
119
+ "bos_token": "<s>",
120
+ "chat_template": "{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}",
121
+ "clean_up_tokenization_spaces": false,
122
+ "eos_token": "<|endoftext|>",
123
+ "legacy": false,
124
+ "model_max_length": 131072,
125
+ "pad_token": "<|endoftext|>",
126
+ "padding_side": "left",
127
+ "sp_model_kwargs": {},
128
+ "tokenizer_class": "LlamaTokenizer",
129
+ "unk_token": "<unk>",
130
+ "use_default_system_prompt": false
131
+ }