diff --git a/README.md b/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..cacd7c317b49e2ff5c8009b286ccab8ed385251f
--- /dev/null
+++ b/README.md
@@ -0,0 +1,201 @@
+---
+library_name: transformers
+tags: []
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+This is the model card of a 🤗 transformers model that has been pushed on the Hub. This model card has been automatically generated.
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure 
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+
+
diff --git a/config.json b/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..35025e844aa2c26b5cbac0edf1d841f25d7c484e
--- /dev/null
+++ b/config.json
@@ -0,0 +1,31 @@
+{
+  "_name_or_path": "mistral-community/Mixtral-8x22B-v0.1",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 6144,
+  "initializer_range": 0.02,
+  "intermediate_size": 16384,
+  "max_position_embeddings": 65536,
+  "model_type": "mixtral",
+  "num_attention_heads": 48,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 56,
+  "num_key_value_heads": 8,
+  "num_local_experts": 8,
+  "output_router_logits": false,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000,
+  "router_aux_loss_coef": 0.001,
+  "router_jitter_noise": 0.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.39.3",
+  "use_cache": true,
+  "vocab_size": 32000
+}
diff --git a/generation_config.json b/generation_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b8e05e6319a87d81bccf1bc16991713250a7dde7
--- /dev/null
+++ b/generation_config.json
@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.39.3"
+}
diff --git a/model-00001-of-00059.safetensors b/model-00001-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..40e3e8536a4de1be74bf6cbd0aa42fc850dccd66
--- /dev/null
+++ b/model-00001-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da21a4930a736d1df65e61f9985be95ac9fec38b317fcd4063126a6799eb8945
+size 4998663672
diff --git a/model-00002-of-00059.safetensors b/model-00002-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d530392beae4e18e616822c486a9a39d2e2662d4
--- /dev/null
+++ b/model-00002-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abe78f96dc9b1b14630819bd978b549e3649325443e63513e6e38df59913af2e
+size 4806799088
diff --git a/model-00003-of-00059.safetensors b/model-00003-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..fe8c4404ac8cdcfdfb9fd8a4153475de2b4e1910
--- /dev/null
+++ b/model-00003-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0127023f59c77f32c62117e4be97621f64df2d55d6c84c037afade4f72fdfec
+size 4806799088
diff --git a/model-00004-of-00059.safetensors b/model-00004-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ed138aa8cf4ec6b6525e342e6a368c357b7fdc39
--- /dev/null
+++ b/model-00004-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db9ecda4e46d15c05d27f3e3b5c682834ae09b09d24b15dc057134ee2c521683
+size 4806799088
diff --git a/model-00005-of-00059.safetensors b/model-00005-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0c2ce92f4d745a83078b7073a9d962ec7f9cada4
--- /dev/null
+++ b/model-00005-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:793555ead8dc75c55313e02aa61c27a3e0da277dd942dc1118d3a9f7ef9be940
+size 4806799088
diff --git a/model-00006-of-00059.safetensors b/model-00006-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..164b591cff76c1f52d24dc9af6889c60dcbf8899
--- /dev/null
+++ b/model-00006-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:154c4f336a82086cd0e400c8f259f972d50a332df736a33ae6247dbbfc58f978
+size 4806799088
diff --git a/model-00007-of-00059.safetensors b/model-00007-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..5f3f48adf435abe168bf446058adaf97047c2f02
--- /dev/null
+++ b/model-00007-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:383276fd569779fd2c1e57d54d8e15d7d406be0d7f2d392c1bb7863cfda1f5d1
+size 4806799088
diff --git a/model-00008-of-00059.safetensors b/model-00008-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c29609d50388c41a6f02d47224a9b9a31b081678
--- /dev/null
+++ b/model-00008-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b6303431cf14c775f2bb00492619f4379708aec18ce661cc6ecbdd63f659c3c
+size 4806799088
diff --git a/model-00009-of-00059.safetensors b/model-00009-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..fd327b51db4f8405cd393f1b610ebe27550296a7
--- /dev/null
+++ b/model-00009-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53c3add01ab5d85db0162264eb0ca8595b4cef91ab107a3854dc938b426678b0
+size 4806799088
diff --git a/model-00010-of-00059.safetensors b/model-00010-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b8750b5cfeec97b9c27c0aaeff4d2e9f0f64695b
--- /dev/null
+++ b/model-00010-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da7687f9487f424713dd60865f3d7b9e194e4a66a0c8e8bc9be8ce7b6e0ac19f
+size 4806799088
diff --git a/model-00011-of-00059.safetensors b/model-00011-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b8a5e3d419f2658f74efc58f1caa93d3be4a7cea
--- /dev/null
+++ b/model-00011-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e97f3d447651c20289d98c2454716e1dc699c9c807bfccea69c205ab70e6e731
+size 4806799112
diff --git a/model-00012-of-00059.safetensors b/model-00012-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e995458073de9eaa3165f204fde9090f9732b1a3
--- /dev/null
+++ b/model-00012-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:644d0b89d18aba51a36294e6a4404189ae2b9de10ab56213f04ab36deeaa7e37
+size 4806799120
diff --git a/model-00013-of-00059.safetensors b/model-00013-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..34ef3c758d4c4d4fec1ec9c09bd63a1f24c70b60
--- /dev/null
+++ b/model-00013-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2d5b8be1d79234208ad8c12fcd271846c68e9d14ee4785b84d474dce824d160
+size 4806799120
diff --git a/model-00014-of-00059.safetensors b/model-00014-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..6b5efda59b8a7cb045fca1162c8504b1c3565920
--- /dev/null
+++ b/model-00014-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f4a5b77bb19465a345fff1dce9a8a6a74a3f43f03dc7583ae13f2b92430dedf
+size 4806799120
diff --git a/model-00015-of-00059.safetensors b/model-00015-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c965892c9c884463d66ace24a861cada5cd0e5a7
--- /dev/null
+++ b/model-00015-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8384d987838d38ed44d8052525fe81202cdc8a82f92bbce2ef40408fd616e48
+size 4806799120
diff --git a/model-00016-of-00059.safetensors b/model-00016-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c1e925ee4b8a929367a7d13fe373e47686503d05
--- /dev/null
+++ b/model-00016-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57e5633662da492acb69870e4cce11219fe6ef6635e1a19079df31fdea6327b3
+size 4806799120
diff --git a/model-00017-of-00059.safetensors b/model-00017-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..7492bd336e394d18a3ecedbfe7880708f8aaae65
--- /dev/null
+++ b/model-00017-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a84aab051ecd8cde8580965f967c6c8be301ee4792a5ceeb8b1258bf75c83a7
+size 4806799120
diff --git a/model-00018-of-00059.safetensors b/model-00018-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0ae5a690b278a78503f0b2a34d017bf17455c245
--- /dev/null
+++ b/model-00018-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:456f9c711d73c40226b4167bb6a88f470a65dddbc4bfba8fba70b9620d40b1c4
+size 4806799120
diff --git a/model-00019-of-00059.safetensors b/model-00019-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..f545b7a5b62ff4180fb6f54a6ae592f140a8113b
--- /dev/null
+++ b/model-00019-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3d961e7b31bf025189173dcfc4f867cb367a7f28932cdd5c8ede82c8ebf0146
+size 4806799120
diff --git a/model-00020-of-00059.safetensors b/model-00020-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..26bf54bc470e87fcb39eefefe500f5fb3a2f002f
--- /dev/null
+++ b/model-00020-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c1741f910777227f9b48081de1467a3168cf66757be193514c887c27e1b34b2
+size 4806799120
diff --git a/model-00021-of-00059.safetensors b/model-00021-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..aced131345146ba8291830c28b2a7493c3cfd8cd
--- /dev/null
+++ b/model-00021-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:692baf1d0875e4a80d886097477ebda2acda9a2dac54f92a33ba6f7f2ed7d934
+size 4806799120
diff --git a/model-00022-of-00059.safetensors b/model-00022-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..132e2b8335a995887c2f06e31d16196242b9a971
--- /dev/null
+++ b/model-00022-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8fac567efcccd5b03527d0a7ac81cf241b2ad481535d072e410b7a3d7f8a7001
+size 4806799120
diff --git a/model-00023-of-00059.safetensors b/model-00023-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0c1800414a3cb001db2b0546a4f2551e822586c2
--- /dev/null
+++ b/model-00023-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c44e0dd3ea2262c706b5724f5880fb656deb4162f90861cd05a7b6e9f3427ad3
+size 4806799120
diff --git a/model-00024-of-00059.safetensors b/model-00024-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b4a9e11b037ec5e8fc5d509ab846f5d90f4b1364
--- /dev/null
+++ b/model-00024-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45d633300930d3cae8c5f0a9e1c8ca138262aaf8d104d6623bb0ecb128b69540
+size 4932529832
diff --git a/model-00025-of-00059.safetensors b/model-00025-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e02ad7e16facea643a8914f9ff8f9c9fbab85ab3
--- /dev/null
+++ b/model-00025-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8ae50bc48d1363b0c899f4f8e402faf560829785c6fbf5e9e30c9c19abc66e1
+size 4995542816
diff --git a/model-00026-of-00059.safetensors b/model-00026-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c51c7e9dedf39fc6db9b298e35dbb75a25463cd2
--- /dev/null
+++ b/model-00026-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76b83a25f54bb71cfcef8a07d457b2c092c45b0fd2aad83a0d6f2b064846c682
+size 4995542816
diff --git a/model-00027-of-00059.safetensors b/model-00027-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ea432d5feee2411d7ff9424d57dd2cf873d6c3ee
--- /dev/null
+++ b/model-00027-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84148f266484f93958355de98355825127da971d6b3d940fc9b806fbf36c3f63
+size 4932628256
diff --git a/model-00028-of-00059.safetensors b/model-00028-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..3e947cbef2185deca627bc1b5ab517333171ccb9
--- /dev/null
+++ b/model-00028-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4a3b1cce520cb1c32bfa7038b0476479a85470019febb3d69c130ff3ef280bd
+size 4806774312
diff --git a/model-00029-of-00059.safetensors b/model-00029-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..16a5e93fbbdfe8bf2ea7c5b7e7f013b693692a0d
--- /dev/null
+++ b/model-00029-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06888432cec897e48c8343b93c168100e64ef85b205ba09661bf64dc3d40e54e
+size 4806799120
diff --git a/model-00030-of-00059.safetensors b/model-00030-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..70891827ddbdeb0fdbe215b82ded86d6755a8259
--- /dev/null
+++ b/model-00030-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3aa2ff84625fce5b01772f87f9069045851c42413296c47e11b4c66840ef5ff9
+size 4806799120
diff --git a/model-00031-of-00059.safetensors b/model-00031-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..cdb65c0e0a177da92274d2b5946c81733bc16e6a
--- /dev/null
+++ b/model-00031-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:735c5de881b2c07519ea8f3718c9d349221bdbb472ce7396a386af95efd5f7d4
+size 4806799120
diff --git a/model-00032-of-00059.safetensors b/model-00032-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..16264249ab62584c80a7b678c62dc4025f17dd1c
--- /dev/null
+++ b/model-00032-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba06ea156a527f9603a2f6c10099c256e299e1fdbafd22bf2fbb78d0399c6be2
+size 4806799120
diff --git a/model-00033-of-00059.safetensors b/model-00033-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0e82ad8fb6984d2cf12c18b25ae6d6ac681881b1
--- /dev/null
+++ b/model-00033-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b9171538022939a1645b666533d87c9c27a5997ac7de967ee5af148b46f3211
+size 4806799120
diff --git a/model-00034-of-00059.safetensors b/model-00034-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..31e5503b7b8aa16ed188f4a939bc3b6c3b0b3a19
--- /dev/null
+++ b/model-00034-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b22becd5f131b3afa15e469db962ce87cc953ae65152b97467a406cdc8e0b1db
+size 4806799120
diff --git a/model-00035-of-00059.safetensors b/model-00035-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ea5d74d65e56fbe11c3e25948dbcbdc8c59f50de
--- /dev/null
+++ b/model-00035-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94b0cb22c3bc35663c47d295389ef487a58d9a866705b5e101a2599d33d85291
+size 4806799120
diff --git a/model-00036-of-00059.safetensors b/model-00036-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..3253461bb7bb982ea8f6f725340565774f8a94b5
--- /dev/null
+++ b/model-00036-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af2dcaf564e081a684a16e10d7f235aaf03eea20c9ae3b1bddfa75b38e94a657
+size 4806799120
diff --git a/model-00037-of-00059.safetensors b/model-00037-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ea66f7b3b3ad7fe7d4aeea03e0520e3896a82a22
--- /dev/null
+++ b/model-00037-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00cb419522bcb530e5c2c61d45af739d8a04b7cf429953766e8b406ecc50dc13
+size 4806799120
diff --git a/model-00038-of-00059.safetensors b/model-00038-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b7bb2933c92c9631f06b6be0f3e7fddba7e4e672
--- /dev/null
+++ b/model-00038-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21997bf60cc4810baf01911d47dfc4a0dfd50223f2bdaed7f25fae2179f2b493
+size 4806799120
diff --git a/model-00039-of-00059.safetensors b/model-00039-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..9094711d9be85f23bcd7908691b43f753fd87c17
--- /dev/null
+++ b/model-00039-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddbb0da220f460116056e129e46985cec2aece53e79b6b1150b2d25a671f7327
+size 4806799120
diff --git a/model-00040-of-00059.safetensors b/model-00040-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..bfab29bc02cbc05474fcc502677291f51a5e4385
--- /dev/null
+++ b/model-00040-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a59168997237957ff3010166ab9c2e16603df53e0b1e46398572d816a71b0d7c
+size 4806799120
diff --git a/model-00041-of-00059.safetensors b/model-00041-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..39d9a8f2771306d3ea6115caa32c7824803539bf
--- /dev/null
+++ b/model-00041-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81d253babe1f5867872b137a76812f44dca52864a798afbd79bbec6155623957
+size 4806799120
diff --git a/model-00042-of-00059.safetensors b/model-00042-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..860a830daeeb1ed77e7a7ada4932c00ecefb274b
--- /dev/null
+++ b/model-00042-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fed6e4a0b3f9c50aed76360d965602229ff8a6e7df2497e13f247e4d2b638559
+size 4806799120
diff --git a/model-00043-of-00059.safetensors b/model-00043-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..2e1175341fe6037c9091dd254b73ab186846d3bc
--- /dev/null
+++ b/model-00043-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02b3543059f5707820984811885c4155a1b3501a3d9fb6bef6334d718cfd7f12
+size 4806799120
diff --git a/model-00044-of-00059.safetensors b/model-00044-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b89dc1d99313c2ed4f2dc6c0937cd0da20475f3b
--- /dev/null
+++ b/model-00044-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d82086c5349167ed0ef106fd8c6a0d82fbb3be0d772a9b7be5754d65d9c9994d
+size 4806799120
diff --git a/model-00045-of-00059.safetensors b/model-00045-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..8fc955fe1db1082ee6d9be13c4991d972939e1fd
--- /dev/null
+++ b/model-00045-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54482fed0a531e6c64241d428382463b1064edf9028966b58eb873c95df3d667
+size 4806799120
diff --git a/model-00046-of-00059.safetensors b/model-00046-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..dc28b6a7a968548107e6a4c5b0c494dd7396c2e2
--- /dev/null
+++ b/model-00046-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a339fe9f30f968826fa474eda4a1f27f383eb088ec2084eb8d41b2de8cf86e11
+size 4806799120
diff --git a/model-00047-of-00059.safetensors b/model-00047-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..13555371c3e4dce8d3f750d35a24c03d01eb5a45
--- /dev/null
+++ b/model-00047-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b313080f8ee8c269642cfa887aa487121a854070cd99dd660ffea6eba0ab258d
+size 4806799120
diff --git a/model-00048-of-00059.safetensors b/model-00048-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..02adc5c0749a3e2ffc8d1b2f713d177d3443f220
--- /dev/null
+++ b/model-00048-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c74846b6bfaca2e03ba2491c379afedd3ecae8d02912da0e124eabfb268ddc45
+size 4806799120
diff --git a/model-00049-of-00059.safetensors b/model-00049-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..79b99343c03a13e9b8d5efc200c914b47a018188
--- /dev/null
+++ b/model-00049-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3dba62b974e07e5602e6d4f7d5a373ff115d1154d28f57170d2306a500782cd3
+size 4806799120
diff --git a/model-00050-of-00059.safetensors b/model-00050-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..a840338d6e333e32c338a7e47164021d957eedac
--- /dev/null
+++ b/model-00050-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5654d04a9646beba90c953c65eb4f47d683df1b85b675bec27c60042a5f5171
+size 4806799120
diff --git a/model-00051-of-00059.safetensors b/model-00051-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..721e2d653466fec761a2a43278c25274daa87392
--- /dev/null
+++ b/model-00051-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:059c8e8da6d114e6fbe2f7ca8994d83d214d4330018249268794a47172af6a2c
+size 4806799120
diff --git a/model-00052-of-00059.safetensors b/model-00052-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..118dba06c6c55bd480980fc7bca7db66b029c46c
--- /dev/null
+++ b/model-00052-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df8f92c02fe78a9b829134cdca499921a22083e1748a4971617b06ad29032895
+size 4932529832
diff --git a/model-00053-of-00059.safetensors b/model-00053-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..7bce452587d33ca64d8d32073165afb12792007b
--- /dev/null
+++ b/model-00053-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c9dfd1b9fd5d45da4ddad0058c1a43b1348ba85cb5a4201658dba643385f5b4
+size 4995542816
diff --git a/model-00054-of-00059.safetensors b/model-00054-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..123083b8557ac301e3ede2104e8d0da9151218e1
--- /dev/null
+++ b/model-00054-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7503983e99838ed784feb593509d4fb4445445145edc93ac46ef27d5f525e394
+size 4995542816
diff --git a/model-00055-of-00059.safetensors b/model-00055-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..6a8cfefa248b7cda5eb8df52d7df3c31f7da9ab5
--- /dev/null
+++ b/model-00055-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edb59e670c20eb0589a176be0ae58d88036335b9249da283d5eea2a51d42a347
+size 4932628256
diff --git a/model-00056-of-00059.safetensors b/model-00056-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..efd2aa91723516a0f70c80b9b2464a8659e6dc4e
--- /dev/null
+++ b/model-00056-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5aa1365cceb22fd1d816ec8350440950a29a7bafa466322edd72bb4507a36a64
+size 4806774312
diff --git a/model-00057-of-00059.safetensors b/model-00057-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..2dacde0ae9da88f7992797bde5e3c7ac01831c6a
--- /dev/null
+++ b/model-00057-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23228221f2f2168da0ffdc4b3e12fb2bbdc6a88cbcdc755602e3cd45979fad1e
+size 4806799120
diff --git a/model-00058-of-00059.safetensors b/model-00058-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..f7bb73dc22af9edba58a29ffce227e13d73a09d6
--- /dev/null
+++ b/model-00058-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ad84b44d421b12e2072c2a447cd456fb817579f78cfa9c9bf99b643fe85cebc
+size 4806799120
diff --git a/model-00059-of-00059.safetensors b/model-00059-of-00059.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..7396db57f7addc0e6b3109fc19a0f5085bfe73cb
--- /dev/null
+++ b/model-00059-of-00059.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b266b369754c4b2ddf57500b157446d95b2381db23f2163f4c68ec3372fa4e3b
+size 997233472
diff --git a/model.safetensors.index.json b/model.safetensors.index.json
new file mode 100644
index 0000000000000000000000000000000000000000..805c18819ee3c4a51164b634df4f4033c74583ad
--- /dev/null
+++ b/model.safetensors.index.json
@@ -0,0 +1,1746 @@
+{
+  "metadata": {
+    "total_size": 281241268224
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00059-of-00059.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00059.safetensors",
+    "model.layers.0.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00059.safetensors",
+    "model.layers.0.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00059.safetensors",
+    "model.layers.0.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00059.safetensors",
+    "model.layers.0.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00059.safetensors",
+    "model.layers.0.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00059.safetensors",
+    "model.layers.0.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00059.safetensors",
+    "model.layers.0.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00059.safetensors",
+    "model.layers.0.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00059.safetensors",
+    "model.layers.0.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00059.safetensors",
+    "model.layers.0.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00059.safetensors",
+    "model.layers.0.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00059.safetensors",
+    "model.layers.0.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00059.safetensors",
+    "model.layers.0.block_sparse_moe.experts.4.w1.weight": "model-00001-of-00059.safetensors",
+    "model.layers.0.block_sparse_moe.experts.4.w2.weight": "model-00001-of-00059.safetensors",
+    "model.layers.0.block_sparse_moe.experts.4.w3.weight": "model-00001-of-00059.safetensors",
+    "model.layers.0.block_sparse_moe.experts.5.w1.weight": "model-00001-of-00059.safetensors",
+    "model.layers.0.block_sparse_moe.experts.5.w2.weight": "model-00001-of-00059.safetensors",
+    "model.layers.0.block_sparse_moe.experts.5.w3.weight": "model-00001-of-00059.safetensors",
+    "model.layers.0.block_sparse_moe.experts.6.w1.weight": "model-00001-of-00059.safetensors",
+    "model.layers.0.block_sparse_moe.experts.6.w2.weight": "model-00001-of-00059.safetensors",
+    "model.layers.0.block_sparse_moe.experts.6.w3.weight": "model-00001-of-00059.safetensors",
+    "model.layers.0.block_sparse_moe.experts.7.w1.weight": "model-00001-of-00059.safetensors",
+    "model.layers.0.block_sparse_moe.experts.7.w2.weight": "model-00002-of-00059.safetensors",
+    "model.layers.0.block_sparse_moe.experts.7.w3.weight": "model-00002-of-00059.safetensors",
+    "model.layers.0.block_sparse_moe.gate.weight": "model-00001-of-00059.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00002-of-00059.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00002-of-00059.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00059.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00059.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00059.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00059.safetensors",
+    "model.layers.1.block_sparse_moe.experts.0.w1.weight": "model-00002-of-00059.safetensors",
+    "model.layers.1.block_sparse_moe.experts.0.w2.weight": "model-00002-of-00059.safetensors",
+    "model.layers.1.block_sparse_moe.experts.0.w3.weight": "model-00002-of-00059.safetensors",
+    "model.layers.1.block_sparse_moe.experts.1.w1.weight": "model-00002-of-00059.safetensors",
+    "model.layers.1.block_sparse_moe.experts.1.w2.weight": "model-00002-of-00059.safetensors",
+    "model.layers.1.block_sparse_moe.experts.1.w3.weight": "model-00002-of-00059.safetensors",
+    "model.layers.1.block_sparse_moe.experts.2.w1.weight": "model-00002-of-00059.safetensors",
+    "model.layers.1.block_sparse_moe.experts.2.w2.weight": "model-00002-of-00059.safetensors",
+    "model.layers.1.block_sparse_moe.experts.2.w3.weight": "model-00002-of-00059.safetensors",
+    "model.layers.1.block_sparse_moe.experts.3.w1.weight": "model-00002-of-00059.safetensors",
+    "model.layers.1.block_sparse_moe.experts.3.w2.weight": "model-00002-of-00059.safetensors",
+    "model.layers.1.block_sparse_moe.experts.3.w3.weight": "model-00002-of-00059.safetensors",
+    "model.layers.1.block_sparse_moe.experts.4.w1.weight": "model-00002-of-00059.safetensors",
+    "model.layers.1.block_sparse_moe.experts.4.w2.weight": "model-00002-of-00059.safetensors",
+    "model.layers.1.block_sparse_moe.experts.4.w3.weight": "model-00002-of-00059.safetensors",
+    "model.layers.1.block_sparse_moe.experts.5.w1.weight": "model-00002-of-00059.safetensors",
+    "model.layers.1.block_sparse_moe.experts.5.w2.weight": "model-00002-of-00059.safetensors",
+    "model.layers.1.block_sparse_moe.experts.5.w3.weight": "model-00002-of-00059.safetensors",
+    "model.layers.1.block_sparse_moe.experts.6.w1.weight": "model-00002-of-00059.safetensors",
+    "model.layers.1.block_sparse_moe.experts.6.w2.weight": "model-00002-of-00059.safetensors",
+    "model.layers.1.block_sparse_moe.experts.6.w3.weight": "model-00002-of-00059.safetensors",
+    "model.layers.1.block_sparse_moe.experts.7.w1.weight": "model-00003-of-00059.safetensors",
+    "model.layers.1.block_sparse_moe.experts.7.w2.weight": "model-00003-of-00059.safetensors",
+    "model.layers.1.block_sparse_moe.experts.7.w3.weight": "model-00003-of-00059.safetensors",
+    "model.layers.1.block_sparse_moe.gate.weight": "model-00002-of-00059.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00003-of-00059.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00003-of-00059.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00002-of-00059.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00002-of-00059.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00002-of-00059.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00002-of-00059.safetensors",
+    "model.layers.10.block_sparse_moe.experts.0.w1.weight": "model-00011-of-00059.safetensors",
+    "model.layers.10.block_sparse_moe.experts.0.w2.weight": "model-00011-of-00059.safetensors",
+    "model.layers.10.block_sparse_moe.experts.0.w3.weight": "model-00011-of-00059.safetensors",
+    "model.layers.10.block_sparse_moe.experts.1.w1.weight": "model-00011-of-00059.safetensors",
+    "model.layers.10.block_sparse_moe.experts.1.w2.weight": "model-00011-of-00059.safetensors",
+    "model.layers.10.block_sparse_moe.experts.1.w3.weight": "model-00011-of-00059.safetensors",
+    "model.layers.10.block_sparse_moe.experts.2.w1.weight": "model-00011-of-00059.safetensors",
+    "model.layers.10.block_sparse_moe.experts.2.w2.weight": "model-00011-of-00059.safetensors",
+    "model.layers.10.block_sparse_moe.experts.2.w3.weight": "model-00011-of-00059.safetensors",
+    "model.layers.10.block_sparse_moe.experts.3.w1.weight": "model-00011-of-00059.safetensors",
+    "model.layers.10.block_sparse_moe.experts.3.w2.weight": "model-00011-of-00059.safetensors",
+    "model.layers.10.block_sparse_moe.experts.3.w3.weight": "model-00011-of-00059.safetensors",
+    "model.layers.10.block_sparse_moe.experts.4.w1.weight": "model-00012-of-00059.safetensors",
+    "model.layers.10.block_sparse_moe.experts.4.w2.weight": "model-00012-of-00059.safetensors",
+    "model.layers.10.block_sparse_moe.experts.4.w3.weight": "model-00012-of-00059.safetensors",
+    "model.layers.10.block_sparse_moe.experts.5.w1.weight": "model-00012-of-00059.safetensors",
+    "model.layers.10.block_sparse_moe.experts.5.w2.weight": "model-00012-of-00059.safetensors",
+    "model.layers.10.block_sparse_moe.experts.5.w3.weight": "model-00012-of-00059.safetensors",
+    "model.layers.10.block_sparse_moe.experts.6.w1.weight": "model-00012-of-00059.safetensors",
+    "model.layers.10.block_sparse_moe.experts.6.w2.weight": "model-00012-of-00059.safetensors",
+    "model.layers.10.block_sparse_moe.experts.6.w3.weight": "model-00012-of-00059.safetensors",
+    "model.layers.10.block_sparse_moe.experts.7.w1.weight": "model-00012-of-00059.safetensors",
+    "model.layers.10.block_sparse_moe.experts.7.w2.weight": "model-00012-of-00059.safetensors",
+    "model.layers.10.block_sparse_moe.experts.7.w3.weight": "model-00012-of-00059.safetensors",
+    "model.layers.10.block_sparse_moe.gate.weight": "model-00011-of-00059.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00012-of-00059.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00012-of-00059.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00011-of-00059.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00011-of-00059.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00011-of-00059.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00011-of-00059.safetensors",
+    "model.layers.11.block_sparse_moe.experts.0.w1.weight": "model-00012-of-00059.safetensors",
+    "model.layers.11.block_sparse_moe.experts.0.w2.weight": "model-00012-of-00059.safetensors",
+    "model.layers.11.block_sparse_moe.experts.0.w3.weight": "model-00012-of-00059.safetensors",
+    "model.layers.11.block_sparse_moe.experts.1.w1.weight": "model-00012-of-00059.safetensors",
+    "model.layers.11.block_sparse_moe.experts.1.w2.weight": "model-00012-of-00059.safetensors",
+    "model.layers.11.block_sparse_moe.experts.1.w3.weight": "model-00012-of-00059.safetensors",
+    "model.layers.11.block_sparse_moe.experts.2.w1.weight": "model-00012-of-00059.safetensors",
+    "model.layers.11.block_sparse_moe.experts.2.w2.weight": "model-00012-of-00059.safetensors",
+    "model.layers.11.block_sparse_moe.experts.2.w3.weight": "model-00012-of-00059.safetensors",
+    "model.layers.11.block_sparse_moe.experts.3.w1.weight": "model-00012-of-00059.safetensors",
+    "model.layers.11.block_sparse_moe.experts.3.w2.weight": "model-00012-of-00059.safetensors",
+    "model.layers.11.block_sparse_moe.experts.3.w3.weight": "model-00013-of-00059.safetensors",
+    "model.layers.11.block_sparse_moe.experts.4.w1.weight": "model-00013-of-00059.safetensors",
+    "model.layers.11.block_sparse_moe.experts.4.w2.weight": "model-00013-of-00059.safetensors",
+    "model.layers.11.block_sparse_moe.experts.4.w3.weight": "model-00013-of-00059.safetensors",
+    "model.layers.11.block_sparse_moe.experts.5.w1.weight": "model-00013-of-00059.safetensors",
+    "model.layers.11.block_sparse_moe.experts.5.w2.weight": "model-00013-of-00059.safetensors",
+    "model.layers.11.block_sparse_moe.experts.5.w3.weight": "model-00013-of-00059.safetensors",
+    "model.layers.11.block_sparse_moe.experts.6.w1.weight": "model-00013-of-00059.safetensors",
+    "model.layers.11.block_sparse_moe.experts.6.w2.weight": "model-00013-of-00059.safetensors",
+    "model.layers.11.block_sparse_moe.experts.6.w3.weight": "model-00013-of-00059.safetensors",
+    "model.layers.11.block_sparse_moe.experts.7.w1.weight": "model-00013-of-00059.safetensors",
+    "model.layers.11.block_sparse_moe.experts.7.w2.weight": "model-00013-of-00059.safetensors",
+    "model.layers.11.block_sparse_moe.experts.7.w3.weight": "model-00013-of-00059.safetensors",
+    "model.layers.11.block_sparse_moe.gate.weight": "model-00012-of-00059.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00013-of-00059.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00013-of-00059.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00012-of-00059.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00012-of-00059.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00012-of-00059.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00012-of-00059.safetensors",
+    "model.layers.12.block_sparse_moe.experts.0.w1.weight": "model-00013-of-00059.safetensors",
+    "model.layers.12.block_sparse_moe.experts.0.w2.weight": "model-00013-of-00059.safetensors",
+    "model.layers.12.block_sparse_moe.experts.0.w3.weight": "model-00013-of-00059.safetensors",
+    "model.layers.12.block_sparse_moe.experts.1.w1.weight": "model-00013-of-00059.safetensors",
+    "model.layers.12.block_sparse_moe.experts.1.w2.weight": "model-00013-of-00059.safetensors",
+    "model.layers.12.block_sparse_moe.experts.1.w3.weight": "model-00013-of-00059.safetensors",
+    "model.layers.12.block_sparse_moe.experts.2.w1.weight": "model-00013-of-00059.safetensors",
+    "model.layers.12.block_sparse_moe.experts.2.w2.weight": "model-00013-of-00059.safetensors",
+    "model.layers.12.block_sparse_moe.experts.2.w3.weight": "model-00013-of-00059.safetensors",
+    "model.layers.12.block_sparse_moe.experts.3.w1.weight": "model-00013-of-00059.safetensors",
+    "model.layers.12.block_sparse_moe.experts.3.w2.weight": "model-00014-of-00059.safetensors",
+    "model.layers.12.block_sparse_moe.experts.3.w3.weight": "model-00014-of-00059.safetensors",
+    "model.layers.12.block_sparse_moe.experts.4.w1.weight": "model-00014-of-00059.safetensors",
+    "model.layers.12.block_sparse_moe.experts.4.w2.weight": "model-00014-of-00059.safetensors",
+    "model.layers.12.block_sparse_moe.experts.4.w3.weight": "model-00014-of-00059.safetensors",
+    "model.layers.12.block_sparse_moe.experts.5.w1.weight": "model-00014-of-00059.safetensors",
+    "model.layers.12.block_sparse_moe.experts.5.w2.weight": "model-00014-of-00059.safetensors",
+    "model.layers.12.block_sparse_moe.experts.5.w3.weight": "model-00014-of-00059.safetensors",
+    "model.layers.12.block_sparse_moe.experts.6.w1.weight": "model-00014-of-00059.safetensors",
+    "model.layers.12.block_sparse_moe.experts.6.w2.weight": "model-00014-of-00059.safetensors",
+    "model.layers.12.block_sparse_moe.experts.6.w3.weight": "model-00014-of-00059.safetensors",
+    "model.layers.12.block_sparse_moe.experts.7.w1.weight": "model-00014-of-00059.safetensors",
+    "model.layers.12.block_sparse_moe.experts.7.w2.weight": "model-00014-of-00059.safetensors",
+    "model.layers.12.block_sparse_moe.experts.7.w3.weight": "model-00014-of-00059.safetensors",
+    "model.layers.12.block_sparse_moe.gate.weight": "model-00013-of-00059.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00014-of-00059.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00014-of-00059.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00013-of-00059.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00013-of-00059.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00013-of-00059.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00013-of-00059.safetensors",
+    "model.layers.13.block_sparse_moe.experts.0.w1.weight": "model-00014-of-00059.safetensors",
+    "model.layers.13.block_sparse_moe.experts.0.w2.weight": "model-00014-of-00059.safetensors",
+    "model.layers.13.block_sparse_moe.experts.0.w3.weight": "model-00014-of-00059.safetensors",
+    "model.layers.13.block_sparse_moe.experts.1.w1.weight": "model-00014-of-00059.safetensors",
+    "model.layers.13.block_sparse_moe.experts.1.w2.weight": "model-00014-of-00059.safetensors",
+    "model.layers.13.block_sparse_moe.experts.1.w3.weight": "model-00014-of-00059.safetensors",
+    "model.layers.13.block_sparse_moe.experts.2.w1.weight": "model-00014-of-00059.safetensors",
+    "model.layers.13.block_sparse_moe.experts.2.w2.weight": "model-00014-of-00059.safetensors",
+    "model.layers.13.block_sparse_moe.experts.2.w3.weight": "model-00014-of-00059.safetensors",
+    "model.layers.13.block_sparse_moe.experts.3.w1.weight": "model-00015-of-00059.safetensors",
+    "model.layers.13.block_sparse_moe.experts.3.w2.weight": "model-00015-of-00059.safetensors",
+    "model.layers.13.block_sparse_moe.experts.3.w3.weight": "model-00015-of-00059.safetensors",
+    "model.layers.13.block_sparse_moe.experts.4.w1.weight": "model-00015-of-00059.safetensors",
+    "model.layers.13.block_sparse_moe.experts.4.w2.weight": "model-00015-of-00059.safetensors",
+    "model.layers.13.block_sparse_moe.experts.4.w3.weight": "model-00015-of-00059.safetensors",
+    "model.layers.13.block_sparse_moe.experts.5.w1.weight": "model-00015-of-00059.safetensors",
+    "model.layers.13.block_sparse_moe.experts.5.w2.weight": "model-00015-of-00059.safetensors",
+    "model.layers.13.block_sparse_moe.experts.5.w3.weight": "model-00015-of-00059.safetensors",
+    "model.layers.13.block_sparse_moe.experts.6.w1.weight": "model-00015-of-00059.safetensors",
+    "model.layers.13.block_sparse_moe.experts.6.w2.weight": "model-00015-of-00059.safetensors",
+    "model.layers.13.block_sparse_moe.experts.6.w3.weight": "model-00015-of-00059.safetensors",
+    "model.layers.13.block_sparse_moe.experts.7.w1.weight": "model-00015-of-00059.safetensors",
+    "model.layers.13.block_sparse_moe.experts.7.w2.weight": "model-00015-of-00059.safetensors",
+    "model.layers.13.block_sparse_moe.experts.7.w3.weight": "model-00015-of-00059.safetensors",
+    "model.layers.13.block_sparse_moe.gate.weight": "model-00014-of-00059.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00015-of-00059.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00015-of-00059.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00014-of-00059.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00014-of-00059.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00014-of-00059.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00014-of-00059.safetensors",
+    "model.layers.14.block_sparse_moe.experts.0.w1.weight": "model-00015-of-00059.safetensors",
+    "model.layers.14.block_sparse_moe.experts.0.w2.weight": "model-00015-of-00059.safetensors",
+    "model.layers.14.block_sparse_moe.experts.0.w3.weight": "model-00015-of-00059.safetensors",
+    "model.layers.14.block_sparse_moe.experts.1.w1.weight": "model-00015-of-00059.safetensors",
+    "model.layers.14.block_sparse_moe.experts.1.w2.weight": "model-00015-of-00059.safetensors",
+    "model.layers.14.block_sparse_moe.experts.1.w3.weight": "model-00015-of-00059.safetensors",
+    "model.layers.14.block_sparse_moe.experts.2.w1.weight": "model-00015-of-00059.safetensors",
+    "model.layers.14.block_sparse_moe.experts.2.w2.weight": "model-00015-of-00059.safetensors",
+    "model.layers.14.block_sparse_moe.experts.2.w3.weight": "model-00016-of-00059.safetensors",
+    "model.layers.14.block_sparse_moe.experts.3.w1.weight": "model-00016-of-00059.safetensors",
+    "model.layers.14.block_sparse_moe.experts.3.w2.weight": "model-00016-of-00059.safetensors",
+    "model.layers.14.block_sparse_moe.experts.3.w3.weight": "model-00016-of-00059.safetensors",
+    "model.layers.14.block_sparse_moe.experts.4.w1.weight": "model-00016-of-00059.safetensors",
+    "model.layers.14.block_sparse_moe.experts.4.w2.weight": "model-00016-of-00059.safetensors",
+    "model.layers.14.block_sparse_moe.experts.4.w3.weight": "model-00016-of-00059.safetensors",
+    "model.layers.14.block_sparse_moe.experts.5.w1.weight": "model-00016-of-00059.safetensors",
+    "model.layers.14.block_sparse_moe.experts.5.w2.weight": "model-00016-of-00059.safetensors",
+    "model.layers.14.block_sparse_moe.experts.5.w3.weight": "model-00016-of-00059.safetensors",
+    "model.layers.14.block_sparse_moe.experts.6.w1.weight": "model-00016-of-00059.safetensors",
+    "model.layers.14.block_sparse_moe.experts.6.w2.weight": "model-00016-of-00059.safetensors",
+    "model.layers.14.block_sparse_moe.experts.6.w3.weight": "model-00016-of-00059.safetensors",
+    "model.layers.14.block_sparse_moe.experts.7.w1.weight": "model-00016-of-00059.safetensors",
+    "model.layers.14.block_sparse_moe.experts.7.w2.weight": "model-00016-of-00059.safetensors",
+    "model.layers.14.block_sparse_moe.experts.7.w3.weight": "model-00016-of-00059.safetensors",
+    "model.layers.14.block_sparse_moe.gate.weight": "model-00015-of-00059.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00016-of-00059.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00016-of-00059.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00015-of-00059.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00015-of-00059.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00015-of-00059.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00015-of-00059.safetensors",
+    "model.layers.15.block_sparse_moe.experts.0.w1.weight": "model-00016-of-00059.safetensors",
+    "model.layers.15.block_sparse_moe.experts.0.w2.weight": "model-00016-of-00059.safetensors",
+    "model.layers.15.block_sparse_moe.experts.0.w3.weight": "model-00016-of-00059.safetensors",
+    "model.layers.15.block_sparse_moe.experts.1.w1.weight": "model-00016-of-00059.safetensors",
+    "model.layers.15.block_sparse_moe.experts.1.w2.weight": "model-00016-of-00059.safetensors",
+    "model.layers.15.block_sparse_moe.experts.1.w3.weight": "model-00016-of-00059.safetensors",
+    "model.layers.15.block_sparse_moe.experts.2.w1.weight": "model-00016-of-00059.safetensors",
+    "model.layers.15.block_sparse_moe.experts.2.w2.weight": "model-00017-of-00059.safetensors",
+    "model.layers.15.block_sparse_moe.experts.2.w3.weight": "model-00017-of-00059.safetensors",
+    "model.layers.15.block_sparse_moe.experts.3.w1.weight": "model-00017-of-00059.safetensors",
+    "model.layers.15.block_sparse_moe.experts.3.w2.weight": "model-00017-of-00059.safetensors",
+    "model.layers.15.block_sparse_moe.experts.3.w3.weight": "model-00017-of-00059.safetensors",
+    "model.layers.15.block_sparse_moe.experts.4.w1.weight": "model-00017-of-00059.safetensors",
+    "model.layers.15.block_sparse_moe.experts.4.w2.weight": "model-00017-of-00059.safetensors",
+    "model.layers.15.block_sparse_moe.experts.4.w3.weight": "model-00017-of-00059.safetensors",
+    "model.layers.15.block_sparse_moe.experts.5.w1.weight": "model-00017-of-00059.safetensors",
+    "model.layers.15.block_sparse_moe.experts.5.w2.weight": "model-00017-of-00059.safetensors",
+    "model.layers.15.block_sparse_moe.experts.5.w3.weight": "model-00017-of-00059.safetensors",
+    "model.layers.15.block_sparse_moe.experts.6.w1.weight": "model-00017-of-00059.safetensors",
+    "model.layers.15.block_sparse_moe.experts.6.w2.weight": "model-00017-of-00059.safetensors",
+    "model.layers.15.block_sparse_moe.experts.6.w3.weight": "model-00017-of-00059.safetensors",
+    "model.layers.15.block_sparse_moe.experts.7.w1.weight": "model-00017-of-00059.safetensors",
+    "model.layers.15.block_sparse_moe.experts.7.w2.weight": "model-00017-of-00059.safetensors",
+    "model.layers.15.block_sparse_moe.experts.7.w3.weight": "model-00017-of-00059.safetensors",
+    "model.layers.15.block_sparse_moe.gate.weight": "model-00016-of-00059.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00017-of-00059.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00017-of-00059.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00016-of-00059.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00016-of-00059.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00016-of-00059.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00016-of-00059.safetensors",
+    "model.layers.16.block_sparse_moe.experts.0.w1.weight": "model-00017-of-00059.safetensors",
+    "model.layers.16.block_sparse_moe.experts.0.w2.weight": "model-00017-of-00059.safetensors",
+    "model.layers.16.block_sparse_moe.experts.0.w3.weight": "model-00017-of-00059.safetensors",
+    "model.layers.16.block_sparse_moe.experts.1.w1.weight": "model-00017-of-00059.safetensors",
+    "model.layers.16.block_sparse_moe.experts.1.w2.weight": "model-00017-of-00059.safetensors",
+    "model.layers.16.block_sparse_moe.experts.1.w3.weight": "model-00017-of-00059.safetensors",
+    "model.layers.16.block_sparse_moe.experts.2.w1.weight": "model-00018-of-00059.safetensors",
+    "model.layers.16.block_sparse_moe.experts.2.w2.weight": "model-00018-of-00059.safetensors",
+    "model.layers.16.block_sparse_moe.experts.2.w3.weight": "model-00018-of-00059.safetensors",
+    "model.layers.16.block_sparse_moe.experts.3.w1.weight": "model-00018-of-00059.safetensors",
+    "model.layers.16.block_sparse_moe.experts.3.w2.weight": "model-00018-of-00059.safetensors",
+    "model.layers.16.block_sparse_moe.experts.3.w3.weight": "model-00018-of-00059.safetensors",
+    "model.layers.16.block_sparse_moe.experts.4.w1.weight": "model-00018-of-00059.safetensors",
+    "model.layers.16.block_sparse_moe.experts.4.w2.weight": "model-00018-of-00059.safetensors",
+    "model.layers.16.block_sparse_moe.experts.4.w3.weight": "model-00018-of-00059.safetensors",
+    "model.layers.16.block_sparse_moe.experts.5.w1.weight": "model-00018-of-00059.safetensors",
+    "model.layers.16.block_sparse_moe.experts.5.w2.weight": "model-00018-of-00059.safetensors",
+    "model.layers.16.block_sparse_moe.experts.5.w3.weight": "model-00018-of-00059.safetensors",
+    "model.layers.16.block_sparse_moe.experts.6.w1.weight": "model-00018-of-00059.safetensors",
+    "model.layers.16.block_sparse_moe.experts.6.w2.weight": "model-00018-of-00059.safetensors",
+    "model.layers.16.block_sparse_moe.experts.6.w3.weight": "model-00018-of-00059.safetensors",
+    "model.layers.16.block_sparse_moe.experts.7.w1.weight": "model-00018-of-00059.safetensors",
+    "model.layers.16.block_sparse_moe.experts.7.w2.weight": "model-00018-of-00059.safetensors",
+    "model.layers.16.block_sparse_moe.experts.7.w3.weight": "model-00018-of-00059.safetensors",
+    "model.layers.16.block_sparse_moe.gate.weight": "model-00017-of-00059.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00018-of-00059.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00018-of-00059.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00017-of-00059.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00017-of-00059.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00017-of-00059.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00017-of-00059.safetensors",
+    "model.layers.17.block_sparse_moe.experts.0.w1.weight": "model-00018-of-00059.safetensors",
+    "model.layers.17.block_sparse_moe.experts.0.w2.weight": "model-00018-of-00059.safetensors",
+    "model.layers.17.block_sparse_moe.experts.0.w3.weight": "model-00018-of-00059.safetensors",
+    "model.layers.17.block_sparse_moe.experts.1.w1.weight": "model-00018-of-00059.safetensors",
+    "model.layers.17.block_sparse_moe.experts.1.w2.weight": "model-00018-of-00059.safetensors",
+    "model.layers.17.block_sparse_moe.experts.1.w3.weight": "model-00019-of-00059.safetensors",
+    "model.layers.17.block_sparse_moe.experts.2.w1.weight": "model-00019-of-00059.safetensors",
+    "model.layers.17.block_sparse_moe.experts.2.w2.weight": "model-00019-of-00059.safetensors",
+    "model.layers.17.block_sparse_moe.experts.2.w3.weight": "model-00019-of-00059.safetensors",
+    "model.layers.17.block_sparse_moe.experts.3.w1.weight": "model-00019-of-00059.safetensors",
+    "model.layers.17.block_sparse_moe.experts.3.w2.weight": "model-00019-of-00059.safetensors",
+    "model.layers.17.block_sparse_moe.experts.3.w3.weight": "model-00019-of-00059.safetensors",
+    "model.layers.17.block_sparse_moe.experts.4.w1.weight": "model-00019-of-00059.safetensors",
+    "model.layers.17.block_sparse_moe.experts.4.w2.weight": "model-00019-of-00059.safetensors",
+    "model.layers.17.block_sparse_moe.experts.4.w3.weight": "model-00019-of-00059.safetensors",
+    "model.layers.17.block_sparse_moe.experts.5.w1.weight": "model-00019-of-00059.safetensors",
+    "model.layers.17.block_sparse_moe.experts.5.w2.weight": "model-00019-of-00059.safetensors",
+    "model.layers.17.block_sparse_moe.experts.5.w3.weight": "model-00019-of-00059.safetensors",
+    "model.layers.17.block_sparse_moe.experts.6.w1.weight": "model-00019-of-00059.safetensors",
+    "model.layers.17.block_sparse_moe.experts.6.w2.weight": "model-00019-of-00059.safetensors",
+    "model.layers.17.block_sparse_moe.experts.6.w3.weight": "model-00019-of-00059.safetensors",
+    "model.layers.17.block_sparse_moe.experts.7.w1.weight": "model-00019-of-00059.safetensors",
+    "model.layers.17.block_sparse_moe.experts.7.w2.weight": "model-00019-of-00059.safetensors",
+    "model.layers.17.block_sparse_moe.experts.7.w3.weight": "model-00019-of-00059.safetensors",
+    "model.layers.17.block_sparse_moe.gate.weight": "model-00018-of-00059.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00019-of-00059.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00019-of-00059.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00018-of-00059.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00018-of-00059.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00018-of-00059.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00018-of-00059.safetensors",
+    "model.layers.18.block_sparse_moe.experts.0.w1.weight": "model-00019-of-00059.safetensors",
+    "model.layers.18.block_sparse_moe.experts.0.w2.weight": "model-00019-of-00059.safetensors",
+    "model.layers.18.block_sparse_moe.experts.0.w3.weight": "model-00019-of-00059.safetensors",
+    "model.layers.18.block_sparse_moe.experts.1.w1.weight": "model-00019-of-00059.safetensors",
+    "model.layers.18.block_sparse_moe.experts.1.w2.weight": "model-00020-of-00059.safetensors",
+    "model.layers.18.block_sparse_moe.experts.1.w3.weight": "model-00020-of-00059.safetensors",
+    "model.layers.18.block_sparse_moe.experts.2.w1.weight": "model-00020-of-00059.safetensors",
+    "model.layers.18.block_sparse_moe.experts.2.w2.weight": "model-00020-of-00059.safetensors",
+    "model.layers.18.block_sparse_moe.experts.2.w3.weight": "model-00020-of-00059.safetensors",
+    "model.layers.18.block_sparse_moe.experts.3.w1.weight": "model-00020-of-00059.safetensors",
+    "model.layers.18.block_sparse_moe.experts.3.w2.weight": "model-00020-of-00059.safetensors",
+    "model.layers.18.block_sparse_moe.experts.3.w3.weight": "model-00020-of-00059.safetensors",
+    "model.layers.18.block_sparse_moe.experts.4.w1.weight": "model-00020-of-00059.safetensors",
+    "model.layers.18.block_sparse_moe.experts.4.w2.weight": "model-00020-of-00059.safetensors",
+    "model.layers.18.block_sparse_moe.experts.4.w3.weight": "model-00020-of-00059.safetensors",
+    "model.layers.18.block_sparse_moe.experts.5.w1.weight": "model-00020-of-00059.safetensors",
+    "model.layers.18.block_sparse_moe.experts.5.w2.weight": "model-00020-of-00059.safetensors",
+    "model.layers.18.block_sparse_moe.experts.5.w3.weight": "model-00020-of-00059.safetensors",
+    "model.layers.18.block_sparse_moe.experts.6.w1.weight": "model-00020-of-00059.safetensors",
+    "model.layers.18.block_sparse_moe.experts.6.w2.weight": "model-00020-of-00059.safetensors",
+    "model.layers.18.block_sparse_moe.experts.6.w3.weight": "model-00020-of-00059.safetensors",
+    "model.layers.18.block_sparse_moe.experts.7.w1.weight": "model-00020-of-00059.safetensors",
+    "model.layers.18.block_sparse_moe.experts.7.w2.weight": "model-00020-of-00059.safetensors",
+    "model.layers.18.block_sparse_moe.experts.7.w3.weight": "model-00020-of-00059.safetensors",
+    "model.layers.18.block_sparse_moe.gate.weight": "model-00019-of-00059.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00020-of-00059.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00020-of-00059.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00019-of-00059.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00019-of-00059.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00019-of-00059.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00019-of-00059.safetensors",
+    "model.layers.19.block_sparse_moe.experts.0.w1.weight": "model-00020-of-00059.safetensors",
+    "model.layers.19.block_sparse_moe.experts.0.w2.weight": "model-00020-of-00059.safetensors",
+    "model.layers.19.block_sparse_moe.experts.0.w3.weight": "model-00020-of-00059.safetensors",
+    "model.layers.19.block_sparse_moe.experts.1.w1.weight": "model-00021-of-00059.safetensors",
+    "model.layers.19.block_sparse_moe.experts.1.w2.weight": "model-00021-of-00059.safetensors",
+    "model.layers.19.block_sparse_moe.experts.1.w3.weight": "model-00021-of-00059.safetensors",
+    "model.layers.19.block_sparse_moe.experts.2.w1.weight": "model-00021-of-00059.safetensors",
+    "model.layers.19.block_sparse_moe.experts.2.w2.weight": "model-00021-of-00059.safetensors",
+    "model.layers.19.block_sparse_moe.experts.2.w3.weight": "model-00021-of-00059.safetensors",
+    "model.layers.19.block_sparse_moe.experts.3.w1.weight": "model-00021-of-00059.safetensors",
+    "model.layers.19.block_sparse_moe.experts.3.w2.weight": "model-00021-of-00059.safetensors",
+    "model.layers.19.block_sparse_moe.experts.3.w3.weight": "model-00021-of-00059.safetensors",
+    "model.layers.19.block_sparse_moe.experts.4.w1.weight": "model-00021-of-00059.safetensors",
+    "model.layers.19.block_sparse_moe.experts.4.w2.weight": "model-00021-of-00059.safetensors",
+    "model.layers.19.block_sparse_moe.experts.4.w3.weight": "model-00021-of-00059.safetensors",
+    "model.layers.19.block_sparse_moe.experts.5.w1.weight": "model-00021-of-00059.safetensors",
+    "model.layers.19.block_sparse_moe.experts.5.w2.weight": "model-00021-of-00059.safetensors",
+    "model.layers.19.block_sparse_moe.experts.5.w3.weight": "model-00021-of-00059.safetensors",
+    "model.layers.19.block_sparse_moe.experts.6.w1.weight": "model-00021-of-00059.safetensors",
+    "model.layers.19.block_sparse_moe.experts.6.w2.weight": "model-00021-of-00059.safetensors",
+    "model.layers.19.block_sparse_moe.experts.6.w3.weight": "model-00021-of-00059.safetensors",
+    "model.layers.19.block_sparse_moe.experts.7.w1.weight": "model-00021-of-00059.safetensors",
+    "model.layers.19.block_sparse_moe.experts.7.w2.weight": "model-00021-of-00059.safetensors",
+    "model.layers.19.block_sparse_moe.experts.7.w3.weight": "model-00021-of-00059.safetensors",
+    "model.layers.19.block_sparse_moe.gate.weight": "model-00020-of-00059.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00021-of-00059.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00021-of-00059.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00020-of-00059.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00020-of-00059.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00020-of-00059.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00020-of-00059.safetensors",
+    "model.layers.2.block_sparse_moe.experts.0.w1.weight": "model-00003-of-00059.safetensors",
+    "model.layers.2.block_sparse_moe.experts.0.w2.weight": "model-00003-of-00059.safetensors",
+    "model.layers.2.block_sparse_moe.experts.0.w3.weight": "model-00003-of-00059.safetensors",
+    "model.layers.2.block_sparse_moe.experts.1.w1.weight": "model-00003-of-00059.safetensors",
+    "model.layers.2.block_sparse_moe.experts.1.w2.weight": "model-00003-of-00059.safetensors",
+    "model.layers.2.block_sparse_moe.experts.1.w3.weight": "model-00003-of-00059.safetensors",
+    "model.layers.2.block_sparse_moe.experts.2.w1.weight": "model-00003-of-00059.safetensors",
+    "model.layers.2.block_sparse_moe.experts.2.w2.weight": "model-00003-of-00059.safetensors",
+    "model.layers.2.block_sparse_moe.experts.2.w3.weight": "model-00003-of-00059.safetensors",
+    "model.layers.2.block_sparse_moe.experts.3.w1.weight": "model-00003-of-00059.safetensors",
+    "model.layers.2.block_sparse_moe.experts.3.w2.weight": "model-00003-of-00059.safetensors",
+    "model.layers.2.block_sparse_moe.experts.3.w3.weight": "model-00003-of-00059.safetensors",
+    "model.layers.2.block_sparse_moe.experts.4.w1.weight": "model-00003-of-00059.safetensors",
+    "model.layers.2.block_sparse_moe.experts.4.w2.weight": "model-00003-of-00059.safetensors",
+    "model.layers.2.block_sparse_moe.experts.4.w3.weight": "model-00003-of-00059.safetensors",
+    "model.layers.2.block_sparse_moe.experts.5.w1.weight": "model-00003-of-00059.safetensors",
+    "model.layers.2.block_sparse_moe.experts.5.w2.weight": "model-00003-of-00059.safetensors",
+    "model.layers.2.block_sparse_moe.experts.5.w3.weight": "model-00003-of-00059.safetensors",
+    "model.layers.2.block_sparse_moe.experts.6.w1.weight": "model-00003-of-00059.safetensors",
+    "model.layers.2.block_sparse_moe.experts.6.w2.weight": "model-00003-of-00059.safetensors",
+    "model.layers.2.block_sparse_moe.experts.6.w3.weight": "model-00004-of-00059.safetensors",
+    "model.layers.2.block_sparse_moe.experts.7.w1.weight": "model-00004-of-00059.safetensors",
+    "model.layers.2.block_sparse_moe.experts.7.w2.weight": "model-00004-of-00059.safetensors",
+    "model.layers.2.block_sparse_moe.experts.7.w3.weight": "model-00004-of-00059.safetensors",
+    "model.layers.2.block_sparse_moe.gate.weight": "model-00003-of-00059.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00004-of-00059.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00004-of-00059.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00003-of-00059.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00003-of-00059.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00003-of-00059.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00003-of-00059.safetensors",
+    "model.layers.20.block_sparse_moe.experts.0.w1.weight": "model-00021-of-00059.safetensors",
+    "model.layers.20.block_sparse_moe.experts.0.w2.weight": "model-00021-of-00059.safetensors",
+    "model.layers.20.block_sparse_moe.experts.0.w3.weight": "model-00022-of-00059.safetensors",
+    "model.layers.20.block_sparse_moe.experts.1.w1.weight": "model-00022-of-00059.safetensors",
+    "model.layers.20.block_sparse_moe.experts.1.w2.weight": "model-00022-of-00059.safetensors",
+    "model.layers.20.block_sparse_moe.experts.1.w3.weight": "model-00022-of-00059.safetensors",
+    "model.layers.20.block_sparse_moe.experts.2.w1.weight": "model-00022-of-00059.safetensors",
+    "model.layers.20.block_sparse_moe.experts.2.w2.weight": "model-00022-of-00059.safetensors",
+    "model.layers.20.block_sparse_moe.experts.2.w3.weight": "model-00022-of-00059.safetensors",
+    "model.layers.20.block_sparse_moe.experts.3.w1.weight": "model-00022-of-00059.safetensors",
+    "model.layers.20.block_sparse_moe.experts.3.w2.weight": "model-00022-of-00059.safetensors",
+    "model.layers.20.block_sparse_moe.experts.3.w3.weight": "model-00022-of-00059.safetensors",
+    "model.layers.20.block_sparse_moe.experts.4.w1.weight": "model-00022-of-00059.safetensors",
+    "model.layers.20.block_sparse_moe.experts.4.w2.weight": "model-00022-of-00059.safetensors",
+    "model.layers.20.block_sparse_moe.experts.4.w3.weight": "model-00022-of-00059.safetensors",
+    "model.layers.20.block_sparse_moe.experts.5.w1.weight": "model-00022-of-00059.safetensors",
+    "model.layers.20.block_sparse_moe.experts.5.w2.weight": "model-00022-of-00059.safetensors",
+    "model.layers.20.block_sparse_moe.experts.5.w3.weight": "model-00022-of-00059.safetensors",
+    "model.layers.20.block_sparse_moe.experts.6.w1.weight": "model-00022-of-00059.safetensors",
+    "model.layers.20.block_sparse_moe.experts.6.w2.weight": "model-00022-of-00059.safetensors",
+    "model.layers.20.block_sparse_moe.experts.6.w3.weight": "model-00022-of-00059.safetensors",
+    "model.layers.20.block_sparse_moe.experts.7.w1.weight": "model-00022-of-00059.safetensors",
+    "model.layers.20.block_sparse_moe.experts.7.w2.weight": "model-00022-of-00059.safetensors",
+    "model.layers.20.block_sparse_moe.experts.7.w3.weight": "model-00022-of-00059.safetensors",
+    "model.layers.20.block_sparse_moe.gate.weight": "model-00021-of-00059.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00022-of-00059.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00022-of-00059.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00021-of-00059.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00021-of-00059.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00021-of-00059.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00021-of-00059.safetensors",
+    "model.layers.21.block_sparse_moe.experts.0.w1.weight": "model-00022-of-00059.safetensors",
+    "model.layers.21.block_sparse_moe.experts.0.w2.weight": "model-00023-of-00059.safetensors",
+    "model.layers.21.block_sparse_moe.experts.0.w3.weight": "model-00023-of-00059.safetensors",
+    "model.layers.21.block_sparse_moe.experts.1.w1.weight": "model-00023-of-00059.safetensors",
+    "model.layers.21.block_sparse_moe.experts.1.w2.weight": "model-00023-of-00059.safetensors",
+    "model.layers.21.block_sparse_moe.experts.1.w3.weight": "model-00023-of-00059.safetensors",
+    "model.layers.21.block_sparse_moe.experts.2.w1.weight": "model-00023-of-00059.safetensors",
+    "model.layers.21.block_sparse_moe.experts.2.w2.weight": "model-00023-of-00059.safetensors",
+    "model.layers.21.block_sparse_moe.experts.2.w3.weight": "model-00023-of-00059.safetensors",
+    "model.layers.21.block_sparse_moe.experts.3.w1.weight": "model-00023-of-00059.safetensors",
+    "model.layers.21.block_sparse_moe.experts.3.w2.weight": "model-00023-of-00059.safetensors",
+    "model.layers.21.block_sparse_moe.experts.3.w3.weight": "model-00023-of-00059.safetensors",
+    "model.layers.21.block_sparse_moe.experts.4.w1.weight": "model-00023-of-00059.safetensors",
+    "model.layers.21.block_sparse_moe.experts.4.w2.weight": "model-00023-of-00059.safetensors",
+    "model.layers.21.block_sparse_moe.experts.4.w3.weight": "model-00023-of-00059.safetensors",
+    "model.layers.21.block_sparse_moe.experts.5.w1.weight": "model-00023-of-00059.safetensors",
+    "model.layers.21.block_sparse_moe.experts.5.w2.weight": "model-00023-of-00059.safetensors",
+    "model.layers.21.block_sparse_moe.experts.5.w3.weight": "model-00023-of-00059.safetensors",
+    "model.layers.21.block_sparse_moe.experts.6.w1.weight": "model-00023-of-00059.safetensors",
+    "model.layers.21.block_sparse_moe.experts.6.w2.weight": "model-00023-of-00059.safetensors",
+    "model.layers.21.block_sparse_moe.experts.6.w3.weight": "model-00023-of-00059.safetensors",
+    "model.layers.21.block_sparse_moe.experts.7.w1.weight": "model-00023-of-00059.safetensors",
+    "model.layers.21.block_sparse_moe.experts.7.w2.weight": "model-00023-of-00059.safetensors",
+    "model.layers.21.block_sparse_moe.experts.7.w3.weight": "model-00023-of-00059.safetensors",
+    "model.layers.21.block_sparse_moe.gate.weight": "model-00022-of-00059.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00023-of-00059.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00023-of-00059.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00022-of-00059.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00022-of-00059.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00022-of-00059.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00022-of-00059.safetensors",
+    "model.layers.22.block_sparse_moe.experts.0.w1.weight": "model-00024-of-00059.safetensors",
+    "model.layers.22.block_sparse_moe.experts.0.w2.weight": "model-00024-of-00059.safetensors",
+    "model.layers.22.block_sparse_moe.experts.0.w3.weight": "model-00024-of-00059.safetensors",
+    "model.layers.22.block_sparse_moe.experts.1.w1.weight": "model-00024-of-00059.safetensors",
+    "model.layers.22.block_sparse_moe.experts.1.w2.weight": "model-00024-of-00059.safetensors",
+    "model.layers.22.block_sparse_moe.experts.1.w3.weight": "model-00024-of-00059.safetensors",
+    "model.layers.22.block_sparse_moe.experts.2.w1.weight": "model-00024-of-00059.safetensors",
+    "model.layers.22.block_sparse_moe.experts.2.w2.weight": "model-00024-of-00059.safetensors",
+    "model.layers.22.block_sparse_moe.experts.2.w3.weight": "model-00024-of-00059.safetensors",
+    "model.layers.22.block_sparse_moe.experts.3.w1.weight": "model-00024-of-00059.safetensors",
+    "model.layers.22.block_sparse_moe.experts.3.w2.weight": "model-00024-of-00059.safetensors",
+    "model.layers.22.block_sparse_moe.experts.3.w3.weight": "model-00024-of-00059.safetensors",
+    "model.layers.22.block_sparse_moe.experts.4.w1.weight": "model-00024-of-00059.safetensors",
+    "model.layers.22.block_sparse_moe.experts.4.w2.weight": "model-00024-of-00059.safetensors",
+    "model.layers.22.block_sparse_moe.experts.4.w3.weight": "model-00024-of-00059.safetensors",
+    "model.layers.22.block_sparse_moe.experts.5.w1.weight": "model-00024-of-00059.safetensors",
+    "model.layers.22.block_sparse_moe.experts.5.w2.weight": "model-00024-of-00059.safetensors",
+    "model.layers.22.block_sparse_moe.experts.5.w3.weight": "model-00024-of-00059.safetensors",
+    "model.layers.22.block_sparse_moe.experts.6.w1.weight": "model-00024-of-00059.safetensors",
+    "model.layers.22.block_sparse_moe.experts.6.w2.weight": "model-00024-of-00059.safetensors",
+    "model.layers.22.block_sparse_moe.experts.6.w3.weight": "model-00024-of-00059.safetensors",
+    "model.layers.22.block_sparse_moe.experts.7.w1.weight": "model-00024-of-00059.safetensors",
+    "model.layers.22.block_sparse_moe.experts.7.w2.weight": "model-00024-of-00059.safetensors",
+    "model.layers.22.block_sparse_moe.experts.7.w3.weight": "model-00024-of-00059.safetensors",
+    "model.layers.22.block_sparse_moe.gate.weight": "model-00023-of-00059.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00024-of-00059.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00024-of-00059.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00023-of-00059.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00023-of-00059.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00023-of-00059.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00023-of-00059.safetensors",
+    "model.layers.23.block_sparse_moe.experts.0.w1.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.block_sparse_moe.experts.0.w2.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.block_sparse_moe.experts.0.w3.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.block_sparse_moe.experts.1.w1.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.block_sparse_moe.experts.1.w2.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.block_sparse_moe.experts.1.w3.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.block_sparse_moe.experts.2.w1.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.block_sparse_moe.experts.2.w2.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.block_sparse_moe.experts.2.w3.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.block_sparse_moe.experts.3.w1.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.block_sparse_moe.experts.3.w2.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.block_sparse_moe.experts.3.w3.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.block_sparse_moe.experts.4.w1.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.block_sparse_moe.experts.4.w2.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.block_sparse_moe.experts.4.w3.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.block_sparse_moe.experts.5.w1.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.block_sparse_moe.experts.5.w2.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.block_sparse_moe.experts.5.w3.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.block_sparse_moe.experts.6.w1.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.block_sparse_moe.experts.6.w2.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.block_sparse_moe.experts.6.w3.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.block_sparse_moe.experts.7.w1.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.block_sparse_moe.experts.7.w2.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.block_sparse_moe.experts.7.w3.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.block_sparse_moe.gate.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00024-of-00059.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00025-of-00059.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00024-of-00059.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00024-of-00059.safetensors",
+    "model.layers.24.block_sparse_moe.experts.0.w1.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.block_sparse_moe.experts.0.w2.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.block_sparse_moe.experts.0.w3.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.block_sparse_moe.experts.1.w1.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.block_sparse_moe.experts.1.w2.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.block_sparse_moe.experts.1.w3.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.block_sparse_moe.experts.2.w1.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.block_sparse_moe.experts.2.w2.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.block_sparse_moe.experts.2.w3.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.block_sparse_moe.experts.3.w1.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.block_sparse_moe.experts.3.w2.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.block_sparse_moe.experts.3.w3.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.block_sparse_moe.experts.4.w1.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.block_sparse_moe.experts.4.w2.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.block_sparse_moe.experts.4.w3.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.block_sparse_moe.experts.5.w1.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.block_sparse_moe.experts.5.w2.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.block_sparse_moe.experts.5.w3.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.block_sparse_moe.experts.6.w1.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.block_sparse_moe.experts.6.w2.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.block_sparse_moe.experts.6.w3.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.block_sparse_moe.experts.7.w1.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.block_sparse_moe.experts.7.w2.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.block_sparse_moe.experts.7.w3.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.block_sparse_moe.gate.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00025-of-00059.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00026-of-00059.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00025-of-00059.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00026-of-00059.safetensors",
+    "model.layers.25.block_sparse_moe.experts.0.w1.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.block_sparse_moe.experts.0.w2.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.block_sparse_moe.experts.0.w3.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.block_sparse_moe.experts.1.w1.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.block_sparse_moe.experts.1.w2.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.block_sparse_moe.experts.1.w3.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.block_sparse_moe.experts.2.w1.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.block_sparse_moe.experts.2.w2.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.block_sparse_moe.experts.2.w3.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.block_sparse_moe.experts.3.w1.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.block_sparse_moe.experts.3.w2.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.block_sparse_moe.experts.3.w3.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.block_sparse_moe.experts.4.w1.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.block_sparse_moe.experts.4.w2.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.block_sparse_moe.experts.4.w3.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.block_sparse_moe.experts.5.w1.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.block_sparse_moe.experts.5.w2.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.block_sparse_moe.experts.5.w3.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.block_sparse_moe.experts.6.w1.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.block_sparse_moe.experts.6.w2.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.block_sparse_moe.experts.6.w3.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.block_sparse_moe.experts.7.w1.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.block_sparse_moe.experts.7.w2.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.block_sparse_moe.experts.7.w3.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.block_sparse_moe.gate.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00027-of-00059.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00026-of-00059.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00027-of-00059.safetensors",
+    "model.layers.26.block_sparse_moe.experts.0.w1.weight": "model-00028-of-00059.safetensors",
+    "model.layers.26.block_sparse_moe.experts.0.w2.weight": "model-00028-of-00059.safetensors",
+    "model.layers.26.block_sparse_moe.experts.0.w3.weight": "model-00028-of-00059.safetensors",
+    "model.layers.26.block_sparse_moe.experts.1.w1.weight": "model-00028-of-00059.safetensors",
+    "model.layers.26.block_sparse_moe.experts.1.w2.weight": "model-00028-of-00059.safetensors",
+    "model.layers.26.block_sparse_moe.experts.1.w3.weight": "model-00028-of-00059.safetensors",
+    "model.layers.26.block_sparse_moe.experts.2.w1.weight": "model-00028-of-00059.safetensors",
+    "model.layers.26.block_sparse_moe.experts.2.w2.weight": "model-00028-of-00059.safetensors",
+    "model.layers.26.block_sparse_moe.experts.2.w3.weight": "model-00028-of-00059.safetensors",
+    "model.layers.26.block_sparse_moe.experts.3.w1.weight": "model-00028-of-00059.safetensors",
+    "model.layers.26.block_sparse_moe.experts.3.w2.weight": "model-00028-of-00059.safetensors",
+    "model.layers.26.block_sparse_moe.experts.3.w3.weight": "model-00028-of-00059.safetensors",
+    "model.layers.26.block_sparse_moe.experts.4.w1.weight": "model-00028-of-00059.safetensors",
+    "model.layers.26.block_sparse_moe.experts.4.w2.weight": "model-00028-of-00059.safetensors",
+    "model.layers.26.block_sparse_moe.experts.4.w3.weight": "model-00028-of-00059.safetensors",
+    "model.layers.26.block_sparse_moe.experts.5.w1.weight": "model-00028-of-00059.safetensors",
+    "model.layers.26.block_sparse_moe.experts.5.w2.weight": "model-00028-of-00059.safetensors",
+    "model.layers.26.block_sparse_moe.experts.5.w3.weight": "model-00028-of-00059.safetensors",
+    "model.layers.26.block_sparse_moe.experts.6.w1.weight": "model-00028-of-00059.safetensors",
+    "model.layers.26.block_sparse_moe.experts.6.w2.weight": "model-00028-of-00059.safetensors",
+    "model.layers.26.block_sparse_moe.experts.6.w3.weight": "model-00028-of-00059.safetensors",
+    "model.layers.26.block_sparse_moe.experts.7.w1.weight": "model-00028-of-00059.safetensors",
+    "model.layers.26.block_sparse_moe.experts.7.w2.weight": "model-00028-of-00059.safetensors",
+    "model.layers.26.block_sparse_moe.experts.7.w3.weight": "model-00029-of-00059.safetensors",
+    "model.layers.26.block_sparse_moe.gate.weight": "model-00028-of-00059.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00029-of-00059.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00029-of-00059.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00028-of-00059.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00028-of-00059.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00028-of-00059.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00028-of-00059.safetensors",
+    "model.layers.27.block_sparse_moe.experts.0.w1.weight": "model-00029-of-00059.safetensors",
+    "model.layers.27.block_sparse_moe.experts.0.w2.weight": "model-00029-of-00059.safetensors",
+    "model.layers.27.block_sparse_moe.experts.0.w3.weight": "model-00029-of-00059.safetensors",
+    "model.layers.27.block_sparse_moe.experts.1.w1.weight": "model-00029-of-00059.safetensors",
+    "model.layers.27.block_sparse_moe.experts.1.w2.weight": "model-00029-of-00059.safetensors",
+    "model.layers.27.block_sparse_moe.experts.1.w3.weight": "model-00029-of-00059.safetensors",
+    "model.layers.27.block_sparse_moe.experts.2.w1.weight": "model-00029-of-00059.safetensors",
+    "model.layers.27.block_sparse_moe.experts.2.w2.weight": "model-00029-of-00059.safetensors",
+    "model.layers.27.block_sparse_moe.experts.2.w3.weight": "model-00029-of-00059.safetensors",
+    "model.layers.27.block_sparse_moe.experts.3.w1.weight": "model-00029-of-00059.safetensors",
+    "model.layers.27.block_sparse_moe.experts.3.w2.weight": "model-00029-of-00059.safetensors",
+    "model.layers.27.block_sparse_moe.experts.3.w3.weight": "model-00029-of-00059.safetensors",
+    "model.layers.27.block_sparse_moe.experts.4.w1.weight": "model-00029-of-00059.safetensors",
+    "model.layers.27.block_sparse_moe.experts.4.w2.weight": "model-00029-of-00059.safetensors",
+    "model.layers.27.block_sparse_moe.experts.4.w3.weight": "model-00029-of-00059.safetensors",
+    "model.layers.27.block_sparse_moe.experts.5.w1.weight": "model-00029-of-00059.safetensors",
+    "model.layers.27.block_sparse_moe.experts.5.w2.weight": "model-00029-of-00059.safetensors",
+    "model.layers.27.block_sparse_moe.experts.5.w3.weight": "model-00029-of-00059.safetensors",
+    "model.layers.27.block_sparse_moe.experts.6.w1.weight": "model-00029-of-00059.safetensors",
+    "model.layers.27.block_sparse_moe.experts.6.w2.weight": "model-00029-of-00059.safetensors",
+    "model.layers.27.block_sparse_moe.experts.6.w3.weight": "model-00029-of-00059.safetensors",
+    "model.layers.27.block_sparse_moe.experts.7.w1.weight": "model-00029-of-00059.safetensors",
+    "model.layers.27.block_sparse_moe.experts.7.w2.weight": "model-00030-of-00059.safetensors",
+    "model.layers.27.block_sparse_moe.experts.7.w3.weight": "model-00030-of-00059.safetensors",
+    "model.layers.27.block_sparse_moe.gate.weight": "model-00029-of-00059.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00030-of-00059.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00030-of-00059.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00029-of-00059.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00029-of-00059.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00029-of-00059.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00029-of-00059.safetensors",
+    "model.layers.28.block_sparse_moe.experts.0.w1.weight": "model-00030-of-00059.safetensors",
+    "model.layers.28.block_sparse_moe.experts.0.w2.weight": "model-00030-of-00059.safetensors",
+    "model.layers.28.block_sparse_moe.experts.0.w3.weight": "model-00030-of-00059.safetensors",
+    "model.layers.28.block_sparse_moe.experts.1.w1.weight": "model-00030-of-00059.safetensors",
+    "model.layers.28.block_sparse_moe.experts.1.w2.weight": "model-00030-of-00059.safetensors",
+    "model.layers.28.block_sparse_moe.experts.1.w3.weight": "model-00030-of-00059.safetensors",
+    "model.layers.28.block_sparse_moe.experts.2.w1.weight": "model-00030-of-00059.safetensors",
+    "model.layers.28.block_sparse_moe.experts.2.w2.weight": "model-00030-of-00059.safetensors",
+    "model.layers.28.block_sparse_moe.experts.2.w3.weight": "model-00030-of-00059.safetensors",
+    "model.layers.28.block_sparse_moe.experts.3.w1.weight": "model-00030-of-00059.safetensors",
+    "model.layers.28.block_sparse_moe.experts.3.w2.weight": "model-00030-of-00059.safetensors",
+    "model.layers.28.block_sparse_moe.experts.3.w3.weight": "model-00030-of-00059.safetensors",
+    "model.layers.28.block_sparse_moe.experts.4.w1.weight": "model-00030-of-00059.safetensors",
+    "model.layers.28.block_sparse_moe.experts.4.w2.weight": "model-00030-of-00059.safetensors",
+    "model.layers.28.block_sparse_moe.experts.4.w3.weight": "model-00030-of-00059.safetensors",
+    "model.layers.28.block_sparse_moe.experts.5.w1.weight": "model-00030-of-00059.safetensors",
+    "model.layers.28.block_sparse_moe.experts.5.w2.weight": "model-00030-of-00059.safetensors",
+    "model.layers.28.block_sparse_moe.experts.5.w3.weight": "model-00030-of-00059.safetensors",
+    "model.layers.28.block_sparse_moe.experts.6.w1.weight": "model-00030-of-00059.safetensors",
+    "model.layers.28.block_sparse_moe.experts.6.w2.weight": "model-00030-of-00059.safetensors",
+    "model.layers.28.block_sparse_moe.experts.6.w3.weight": "model-00030-of-00059.safetensors",
+    "model.layers.28.block_sparse_moe.experts.7.w1.weight": "model-00031-of-00059.safetensors",
+    "model.layers.28.block_sparse_moe.experts.7.w2.weight": "model-00031-of-00059.safetensors",
+    "model.layers.28.block_sparse_moe.experts.7.w3.weight": "model-00031-of-00059.safetensors",
+    "model.layers.28.block_sparse_moe.gate.weight": "model-00030-of-00059.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00031-of-00059.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00031-of-00059.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00030-of-00059.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00030-of-00059.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00030-of-00059.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00030-of-00059.safetensors",
+    "model.layers.29.block_sparse_moe.experts.0.w1.weight": "model-00031-of-00059.safetensors",
+    "model.layers.29.block_sparse_moe.experts.0.w2.weight": "model-00031-of-00059.safetensors",
+    "model.layers.29.block_sparse_moe.experts.0.w3.weight": "model-00031-of-00059.safetensors",
+    "model.layers.29.block_sparse_moe.experts.1.w1.weight": "model-00031-of-00059.safetensors",
+    "model.layers.29.block_sparse_moe.experts.1.w2.weight": "model-00031-of-00059.safetensors",
+    "model.layers.29.block_sparse_moe.experts.1.w3.weight": "model-00031-of-00059.safetensors",
+    "model.layers.29.block_sparse_moe.experts.2.w1.weight": "model-00031-of-00059.safetensors",
+    "model.layers.29.block_sparse_moe.experts.2.w2.weight": "model-00031-of-00059.safetensors",
+    "model.layers.29.block_sparse_moe.experts.2.w3.weight": "model-00031-of-00059.safetensors",
+    "model.layers.29.block_sparse_moe.experts.3.w1.weight": "model-00031-of-00059.safetensors",
+    "model.layers.29.block_sparse_moe.experts.3.w2.weight": "model-00031-of-00059.safetensors",
+    "model.layers.29.block_sparse_moe.experts.3.w3.weight": "model-00031-of-00059.safetensors",
+    "model.layers.29.block_sparse_moe.experts.4.w1.weight": "model-00031-of-00059.safetensors",
+    "model.layers.29.block_sparse_moe.experts.4.w2.weight": "model-00031-of-00059.safetensors",
+    "model.layers.29.block_sparse_moe.experts.4.w3.weight": "model-00031-of-00059.safetensors",
+    "model.layers.29.block_sparse_moe.experts.5.w1.weight": "model-00031-of-00059.safetensors",
+    "model.layers.29.block_sparse_moe.experts.5.w2.weight": "model-00031-of-00059.safetensors",
+    "model.layers.29.block_sparse_moe.experts.5.w3.weight": "model-00031-of-00059.safetensors",
+    "model.layers.29.block_sparse_moe.experts.6.w1.weight": "model-00031-of-00059.safetensors",
+    "model.layers.29.block_sparse_moe.experts.6.w2.weight": "model-00031-of-00059.safetensors",
+    "model.layers.29.block_sparse_moe.experts.6.w3.weight": "model-00032-of-00059.safetensors",
+    "model.layers.29.block_sparse_moe.experts.7.w1.weight": "model-00032-of-00059.safetensors",
+    "model.layers.29.block_sparse_moe.experts.7.w2.weight": "model-00032-of-00059.safetensors",
+    "model.layers.29.block_sparse_moe.experts.7.w3.weight": "model-00032-of-00059.safetensors",
+    "model.layers.29.block_sparse_moe.gate.weight": "model-00031-of-00059.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00032-of-00059.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00032-of-00059.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00031-of-00059.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00031-of-00059.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00031-of-00059.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00031-of-00059.safetensors",
+    "model.layers.3.block_sparse_moe.experts.0.w1.weight": "model-00004-of-00059.safetensors",
+    "model.layers.3.block_sparse_moe.experts.0.w2.weight": "model-00004-of-00059.safetensors",
+    "model.layers.3.block_sparse_moe.experts.0.w3.weight": "model-00004-of-00059.safetensors",
+    "model.layers.3.block_sparse_moe.experts.1.w1.weight": "model-00004-of-00059.safetensors",
+    "model.layers.3.block_sparse_moe.experts.1.w2.weight": "model-00004-of-00059.safetensors",
+    "model.layers.3.block_sparse_moe.experts.1.w3.weight": "model-00004-of-00059.safetensors",
+    "model.layers.3.block_sparse_moe.experts.2.w1.weight": "model-00004-of-00059.safetensors",
+    "model.layers.3.block_sparse_moe.experts.2.w2.weight": "model-00004-of-00059.safetensors",
+    "model.layers.3.block_sparse_moe.experts.2.w3.weight": "model-00004-of-00059.safetensors",
+    "model.layers.3.block_sparse_moe.experts.3.w1.weight": "model-00004-of-00059.safetensors",
+    "model.layers.3.block_sparse_moe.experts.3.w2.weight": "model-00004-of-00059.safetensors",
+    "model.layers.3.block_sparse_moe.experts.3.w3.weight": "model-00004-of-00059.safetensors",
+    "model.layers.3.block_sparse_moe.experts.4.w1.weight": "model-00004-of-00059.safetensors",
+    "model.layers.3.block_sparse_moe.experts.4.w2.weight": "model-00004-of-00059.safetensors",
+    "model.layers.3.block_sparse_moe.experts.4.w3.weight": "model-00004-of-00059.safetensors",
+    "model.layers.3.block_sparse_moe.experts.5.w1.weight": "model-00004-of-00059.safetensors",
+    "model.layers.3.block_sparse_moe.experts.5.w2.weight": "model-00004-of-00059.safetensors",
+    "model.layers.3.block_sparse_moe.experts.5.w3.weight": "model-00004-of-00059.safetensors",
+    "model.layers.3.block_sparse_moe.experts.6.w1.weight": "model-00004-of-00059.safetensors",
+    "model.layers.3.block_sparse_moe.experts.6.w2.weight": "model-00005-of-00059.safetensors",
+    "model.layers.3.block_sparse_moe.experts.6.w3.weight": "model-00005-of-00059.safetensors",
+    "model.layers.3.block_sparse_moe.experts.7.w1.weight": "model-00005-of-00059.safetensors",
+    "model.layers.3.block_sparse_moe.experts.7.w2.weight": "model-00005-of-00059.safetensors",
+    "model.layers.3.block_sparse_moe.experts.7.w3.weight": "model-00005-of-00059.safetensors",
+    "model.layers.3.block_sparse_moe.gate.weight": "model-00004-of-00059.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00005-of-00059.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00005-of-00059.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00004-of-00059.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00004-of-00059.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00004-of-00059.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00004-of-00059.safetensors",
+    "model.layers.30.block_sparse_moe.experts.0.w1.weight": "model-00032-of-00059.safetensors",
+    "model.layers.30.block_sparse_moe.experts.0.w2.weight": "model-00032-of-00059.safetensors",
+    "model.layers.30.block_sparse_moe.experts.0.w3.weight": "model-00032-of-00059.safetensors",
+    "model.layers.30.block_sparse_moe.experts.1.w1.weight": "model-00032-of-00059.safetensors",
+    "model.layers.30.block_sparse_moe.experts.1.w2.weight": "model-00032-of-00059.safetensors",
+    "model.layers.30.block_sparse_moe.experts.1.w3.weight": "model-00032-of-00059.safetensors",
+    "model.layers.30.block_sparse_moe.experts.2.w1.weight": "model-00032-of-00059.safetensors",
+    "model.layers.30.block_sparse_moe.experts.2.w2.weight": "model-00032-of-00059.safetensors",
+    "model.layers.30.block_sparse_moe.experts.2.w3.weight": "model-00032-of-00059.safetensors",
+    "model.layers.30.block_sparse_moe.experts.3.w1.weight": "model-00032-of-00059.safetensors",
+    "model.layers.30.block_sparse_moe.experts.3.w2.weight": "model-00032-of-00059.safetensors",
+    "model.layers.30.block_sparse_moe.experts.3.w3.weight": "model-00032-of-00059.safetensors",
+    "model.layers.30.block_sparse_moe.experts.4.w1.weight": "model-00032-of-00059.safetensors",
+    "model.layers.30.block_sparse_moe.experts.4.w2.weight": "model-00032-of-00059.safetensors",
+    "model.layers.30.block_sparse_moe.experts.4.w3.weight": "model-00032-of-00059.safetensors",
+    "model.layers.30.block_sparse_moe.experts.5.w1.weight": "model-00032-of-00059.safetensors",
+    "model.layers.30.block_sparse_moe.experts.5.w2.weight": "model-00032-of-00059.safetensors",
+    "model.layers.30.block_sparse_moe.experts.5.w3.weight": "model-00032-of-00059.safetensors",
+    "model.layers.30.block_sparse_moe.experts.6.w1.weight": "model-00032-of-00059.safetensors",
+    "model.layers.30.block_sparse_moe.experts.6.w2.weight": "model-00033-of-00059.safetensors",
+    "model.layers.30.block_sparse_moe.experts.6.w3.weight": "model-00033-of-00059.safetensors",
+    "model.layers.30.block_sparse_moe.experts.7.w1.weight": "model-00033-of-00059.safetensors",
+    "model.layers.30.block_sparse_moe.experts.7.w2.weight": "model-00033-of-00059.safetensors",
+    "model.layers.30.block_sparse_moe.experts.7.w3.weight": "model-00033-of-00059.safetensors",
+    "model.layers.30.block_sparse_moe.gate.weight": "model-00032-of-00059.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00033-of-00059.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00033-of-00059.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00032-of-00059.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00032-of-00059.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00032-of-00059.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00032-of-00059.safetensors",
+    "model.layers.31.block_sparse_moe.experts.0.w1.weight": "model-00033-of-00059.safetensors",
+    "model.layers.31.block_sparse_moe.experts.0.w2.weight": "model-00033-of-00059.safetensors",
+    "model.layers.31.block_sparse_moe.experts.0.w3.weight": "model-00033-of-00059.safetensors",
+    "model.layers.31.block_sparse_moe.experts.1.w1.weight": "model-00033-of-00059.safetensors",
+    "model.layers.31.block_sparse_moe.experts.1.w2.weight": "model-00033-of-00059.safetensors",
+    "model.layers.31.block_sparse_moe.experts.1.w3.weight": "model-00033-of-00059.safetensors",
+    "model.layers.31.block_sparse_moe.experts.2.w1.weight": "model-00033-of-00059.safetensors",
+    "model.layers.31.block_sparse_moe.experts.2.w2.weight": "model-00033-of-00059.safetensors",
+    "model.layers.31.block_sparse_moe.experts.2.w3.weight": "model-00033-of-00059.safetensors",
+    "model.layers.31.block_sparse_moe.experts.3.w1.weight": "model-00033-of-00059.safetensors",
+    "model.layers.31.block_sparse_moe.experts.3.w2.weight": "model-00033-of-00059.safetensors",
+    "model.layers.31.block_sparse_moe.experts.3.w3.weight": "model-00033-of-00059.safetensors",
+    "model.layers.31.block_sparse_moe.experts.4.w1.weight": "model-00033-of-00059.safetensors",
+    "model.layers.31.block_sparse_moe.experts.4.w2.weight": "model-00033-of-00059.safetensors",
+    "model.layers.31.block_sparse_moe.experts.4.w3.weight": "model-00033-of-00059.safetensors",
+    "model.layers.31.block_sparse_moe.experts.5.w1.weight": "model-00033-of-00059.safetensors",
+    "model.layers.31.block_sparse_moe.experts.5.w2.weight": "model-00033-of-00059.safetensors",
+    "model.layers.31.block_sparse_moe.experts.5.w3.weight": "model-00033-of-00059.safetensors",
+    "model.layers.31.block_sparse_moe.experts.6.w1.weight": "model-00034-of-00059.safetensors",
+    "model.layers.31.block_sparse_moe.experts.6.w2.weight": "model-00034-of-00059.safetensors",
+    "model.layers.31.block_sparse_moe.experts.6.w3.weight": "model-00034-of-00059.safetensors",
+    "model.layers.31.block_sparse_moe.experts.7.w1.weight": "model-00034-of-00059.safetensors",
+    "model.layers.31.block_sparse_moe.experts.7.w2.weight": "model-00034-of-00059.safetensors",
+    "model.layers.31.block_sparse_moe.experts.7.w3.weight": "model-00034-of-00059.safetensors",
+    "model.layers.31.block_sparse_moe.gate.weight": "model-00033-of-00059.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00034-of-00059.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00034-of-00059.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00033-of-00059.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00033-of-00059.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00033-of-00059.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00033-of-00059.safetensors",
+    "model.layers.32.block_sparse_moe.experts.0.w1.weight": "model-00034-of-00059.safetensors",
+    "model.layers.32.block_sparse_moe.experts.0.w2.weight": "model-00034-of-00059.safetensors",
+    "model.layers.32.block_sparse_moe.experts.0.w3.weight": "model-00034-of-00059.safetensors",
+    "model.layers.32.block_sparse_moe.experts.1.w1.weight": "model-00034-of-00059.safetensors",
+    "model.layers.32.block_sparse_moe.experts.1.w2.weight": "model-00034-of-00059.safetensors",
+    "model.layers.32.block_sparse_moe.experts.1.w3.weight": "model-00034-of-00059.safetensors",
+    "model.layers.32.block_sparse_moe.experts.2.w1.weight": "model-00034-of-00059.safetensors",
+    "model.layers.32.block_sparse_moe.experts.2.w2.weight": "model-00034-of-00059.safetensors",
+    "model.layers.32.block_sparse_moe.experts.2.w3.weight": "model-00034-of-00059.safetensors",
+    "model.layers.32.block_sparse_moe.experts.3.w1.weight": "model-00034-of-00059.safetensors",
+    "model.layers.32.block_sparse_moe.experts.3.w2.weight": "model-00034-of-00059.safetensors",
+    "model.layers.32.block_sparse_moe.experts.3.w3.weight": "model-00034-of-00059.safetensors",
+    "model.layers.32.block_sparse_moe.experts.4.w1.weight": "model-00034-of-00059.safetensors",
+    "model.layers.32.block_sparse_moe.experts.4.w2.weight": "model-00034-of-00059.safetensors",
+    "model.layers.32.block_sparse_moe.experts.4.w3.weight": "model-00034-of-00059.safetensors",
+    "model.layers.32.block_sparse_moe.experts.5.w1.weight": "model-00034-of-00059.safetensors",
+    "model.layers.32.block_sparse_moe.experts.5.w2.weight": "model-00034-of-00059.safetensors",
+    "model.layers.32.block_sparse_moe.experts.5.w3.weight": "model-00035-of-00059.safetensors",
+    "model.layers.32.block_sparse_moe.experts.6.w1.weight": "model-00035-of-00059.safetensors",
+    "model.layers.32.block_sparse_moe.experts.6.w2.weight": "model-00035-of-00059.safetensors",
+    "model.layers.32.block_sparse_moe.experts.6.w3.weight": "model-00035-of-00059.safetensors",
+    "model.layers.32.block_sparse_moe.experts.7.w1.weight": "model-00035-of-00059.safetensors",
+    "model.layers.32.block_sparse_moe.experts.7.w2.weight": "model-00035-of-00059.safetensors",
+    "model.layers.32.block_sparse_moe.experts.7.w3.weight": "model-00035-of-00059.safetensors",
+    "model.layers.32.block_sparse_moe.gate.weight": "model-00034-of-00059.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00035-of-00059.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00035-of-00059.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00034-of-00059.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00034-of-00059.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00034-of-00059.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00034-of-00059.safetensors",
+    "model.layers.33.block_sparse_moe.experts.0.w1.weight": "model-00035-of-00059.safetensors",
+    "model.layers.33.block_sparse_moe.experts.0.w2.weight": "model-00035-of-00059.safetensors",
+    "model.layers.33.block_sparse_moe.experts.0.w3.weight": "model-00035-of-00059.safetensors",
+    "model.layers.33.block_sparse_moe.experts.1.w1.weight": "model-00035-of-00059.safetensors",
+    "model.layers.33.block_sparse_moe.experts.1.w2.weight": "model-00035-of-00059.safetensors",
+    "model.layers.33.block_sparse_moe.experts.1.w3.weight": "model-00035-of-00059.safetensors",
+    "model.layers.33.block_sparse_moe.experts.2.w1.weight": "model-00035-of-00059.safetensors",
+    "model.layers.33.block_sparse_moe.experts.2.w2.weight": "model-00035-of-00059.safetensors",
+    "model.layers.33.block_sparse_moe.experts.2.w3.weight": "model-00035-of-00059.safetensors",
+    "model.layers.33.block_sparse_moe.experts.3.w1.weight": "model-00035-of-00059.safetensors",
+    "model.layers.33.block_sparse_moe.experts.3.w2.weight": "model-00035-of-00059.safetensors",
+    "model.layers.33.block_sparse_moe.experts.3.w3.weight": "model-00035-of-00059.safetensors",
+    "model.layers.33.block_sparse_moe.experts.4.w1.weight": "model-00035-of-00059.safetensors",
+    "model.layers.33.block_sparse_moe.experts.4.w2.weight": "model-00035-of-00059.safetensors",
+    "model.layers.33.block_sparse_moe.experts.4.w3.weight": "model-00035-of-00059.safetensors",
+    "model.layers.33.block_sparse_moe.experts.5.w1.weight": "model-00035-of-00059.safetensors",
+    "model.layers.33.block_sparse_moe.experts.5.w2.weight": "model-00036-of-00059.safetensors",
+    "model.layers.33.block_sparse_moe.experts.5.w3.weight": "model-00036-of-00059.safetensors",
+    "model.layers.33.block_sparse_moe.experts.6.w1.weight": "model-00036-of-00059.safetensors",
+    "model.layers.33.block_sparse_moe.experts.6.w2.weight": "model-00036-of-00059.safetensors",
+    "model.layers.33.block_sparse_moe.experts.6.w3.weight": "model-00036-of-00059.safetensors",
+    "model.layers.33.block_sparse_moe.experts.7.w1.weight": "model-00036-of-00059.safetensors",
+    "model.layers.33.block_sparse_moe.experts.7.w2.weight": "model-00036-of-00059.safetensors",
+    "model.layers.33.block_sparse_moe.experts.7.w3.weight": "model-00036-of-00059.safetensors",
+    "model.layers.33.block_sparse_moe.gate.weight": "model-00035-of-00059.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00036-of-00059.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00036-of-00059.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00035-of-00059.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00035-of-00059.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00035-of-00059.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00035-of-00059.safetensors",
+    "model.layers.34.block_sparse_moe.experts.0.w1.weight": "model-00036-of-00059.safetensors",
+    "model.layers.34.block_sparse_moe.experts.0.w2.weight": "model-00036-of-00059.safetensors",
+    "model.layers.34.block_sparse_moe.experts.0.w3.weight": "model-00036-of-00059.safetensors",
+    "model.layers.34.block_sparse_moe.experts.1.w1.weight": "model-00036-of-00059.safetensors",
+    "model.layers.34.block_sparse_moe.experts.1.w2.weight": "model-00036-of-00059.safetensors",
+    "model.layers.34.block_sparse_moe.experts.1.w3.weight": "model-00036-of-00059.safetensors",
+    "model.layers.34.block_sparse_moe.experts.2.w1.weight": "model-00036-of-00059.safetensors",
+    "model.layers.34.block_sparse_moe.experts.2.w2.weight": "model-00036-of-00059.safetensors",
+    "model.layers.34.block_sparse_moe.experts.2.w3.weight": "model-00036-of-00059.safetensors",
+    "model.layers.34.block_sparse_moe.experts.3.w1.weight": "model-00036-of-00059.safetensors",
+    "model.layers.34.block_sparse_moe.experts.3.w2.weight": "model-00036-of-00059.safetensors",
+    "model.layers.34.block_sparse_moe.experts.3.w3.weight": "model-00036-of-00059.safetensors",
+    "model.layers.34.block_sparse_moe.experts.4.w1.weight": "model-00036-of-00059.safetensors",
+    "model.layers.34.block_sparse_moe.experts.4.w2.weight": "model-00036-of-00059.safetensors",
+    "model.layers.34.block_sparse_moe.experts.4.w3.weight": "model-00036-of-00059.safetensors",
+    "model.layers.34.block_sparse_moe.experts.5.w1.weight": "model-00037-of-00059.safetensors",
+    "model.layers.34.block_sparse_moe.experts.5.w2.weight": "model-00037-of-00059.safetensors",
+    "model.layers.34.block_sparse_moe.experts.5.w3.weight": "model-00037-of-00059.safetensors",
+    "model.layers.34.block_sparse_moe.experts.6.w1.weight": "model-00037-of-00059.safetensors",
+    "model.layers.34.block_sparse_moe.experts.6.w2.weight": "model-00037-of-00059.safetensors",
+    "model.layers.34.block_sparse_moe.experts.6.w3.weight": "model-00037-of-00059.safetensors",
+    "model.layers.34.block_sparse_moe.experts.7.w1.weight": "model-00037-of-00059.safetensors",
+    "model.layers.34.block_sparse_moe.experts.7.w2.weight": "model-00037-of-00059.safetensors",
+    "model.layers.34.block_sparse_moe.experts.7.w3.weight": "model-00037-of-00059.safetensors",
+    "model.layers.34.block_sparse_moe.gate.weight": "model-00036-of-00059.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00037-of-00059.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00037-of-00059.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00036-of-00059.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00036-of-00059.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00036-of-00059.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00036-of-00059.safetensors",
+    "model.layers.35.block_sparse_moe.experts.0.w1.weight": "model-00037-of-00059.safetensors",
+    "model.layers.35.block_sparse_moe.experts.0.w2.weight": "model-00037-of-00059.safetensors",
+    "model.layers.35.block_sparse_moe.experts.0.w3.weight": "model-00037-of-00059.safetensors",
+    "model.layers.35.block_sparse_moe.experts.1.w1.weight": "model-00037-of-00059.safetensors",
+    "model.layers.35.block_sparse_moe.experts.1.w2.weight": "model-00037-of-00059.safetensors",
+    "model.layers.35.block_sparse_moe.experts.1.w3.weight": "model-00037-of-00059.safetensors",
+    "model.layers.35.block_sparse_moe.experts.2.w1.weight": "model-00037-of-00059.safetensors",
+    "model.layers.35.block_sparse_moe.experts.2.w2.weight": "model-00037-of-00059.safetensors",
+    "model.layers.35.block_sparse_moe.experts.2.w3.weight": "model-00037-of-00059.safetensors",
+    "model.layers.35.block_sparse_moe.experts.3.w1.weight": "model-00037-of-00059.safetensors",
+    "model.layers.35.block_sparse_moe.experts.3.w2.weight": "model-00037-of-00059.safetensors",
+    "model.layers.35.block_sparse_moe.experts.3.w3.weight": "model-00037-of-00059.safetensors",
+    "model.layers.35.block_sparse_moe.experts.4.w1.weight": "model-00037-of-00059.safetensors",
+    "model.layers.35.block_sparse_moe.experts.4.w2.weight": "model-00037-of-00059.safetensors",
+    "model.layers.35.block_sparse_moe.experts.4.w3.weight": "model-00038-of-00059.safetensors",
+    "model.layers.35.block_sparse_moe.experts.5.w1.weight": "model-00038-of-00059.safetensors",
+    "model.layers.35.block_sparse_moe.experts.5.w2.weight": "model-00038-of-00059.safetensors",
+    "model.layers.35.block_sparse_moe.experts.5.w3.weight": "model-00038-of-00059.safetensors",
+    "model.layers.35.block_sparse_moe.experts.6.w1.weight": "model-00038-of-00059.safetensors",
+    "model.layers.35.block_sparse_moe.experts.6.w2.weight": "model-00038-of-00059.safetensors",
+    "model.layers.35.block_sparse_moe.experts.6.w3.weight": "model-00038-of-00059.safetensors",
+    "model.layers.35.block_sparse_moe.experts.7.w1.weight": "model-00038-of-00059.safetensors",
+    "model.layers.35.block_sparse_moe.experts.7.w2.weight": "model-00038-of-00059.safetensors",
+    "model.layers.35.block_sparse_moe.experts.7.w3.weight": "model-00038-of-00059.safetensors",
+    "model.layers.35.block_sparse_moe.gate.weight": "model-00037-of-00059.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00038-of-00059.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00038-of-00059.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00037-of-00059.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00037-of-00059.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00037-of-00059.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00037-of-00059.safetensors",
+    "model.layers.36.block_sparse_moe.experts.0.w1.weight": "model-00038-of-00059.safetensors",
+    "model.layers.36.block_sparse_moe.experts.0.w2.weight": "model-00038-of-00059.safetensors",
+    "model.layers.36.block_sparse_moe.experts.0.w3.weight": "model-00038-of-00059.safetensors",
+    "model.layers.36.block_sparse_moe.experts.1.w1.weight": "model-00038-of-00059.safetensors",
+    "model.layers.36.block_sparse_moe.experts.1.w2.weight": "model-00038-of-00059.safetensors",
+    "model.layers.36.block_sparse_moe.experts.1.w3.weight": "model-00038-of-00059.safetensors",
+    "model.layers.36.block_sparse_moe.experts.2.w1.weight": "model-00038-of-00059.safetensors",
+    "model.layers.36.block_sparse_moe.experts.2.w2.weight": "model-00038-of-00059.safetensors",
+    "model.layers.36.block_sparse_moe.experts.2.w3.weight": "model-00038-of-00059.safetensors",
+    "model.layers.36.block_sparse_moe.experts.3.w1.weight": "model-00038-of-00059.safetensors",
+    "model.layers.36.block_sparse_moe.experts.3.w2.weight": "model-00038-of-00059.safetensors",
+    "model.layers.36.block_sparse_moe.experts.3.w3.weight": "model-00038-of-00059.safetensors",
+    "model.layers.36.block_sparse_moe.experts.4.w1.weight": "model-00038-of-00059.safetensors",
+    "model.layers.36.block_sparse_moe.experts.4.w2.weight": "model-00039-of-00059.safetensors",
+    "model.layers.36.block_sparse_moe.experts.4.w3.weight": "model-00039-of-00059.safetensors",
+    "model.layers.36.block_sparse_moe.experts.5.w1.weight": "model-00039-of-00059.safetensors",
+    "model.layers.36.block_sparse_moe.experts.5.w2.weight": "model-00039-of-00059.safetensors",
+    "model.layers.36.block_sparse_moe.experts.5.w3.weight": "model-00039-of-00059.safetensors",
+    "model.layers.36.block_sparse_moe.experts.6.w1.weight": "model-00039-of-00059.safetensors",
+    "model.layers.36.block_sparse_moe.experts.6.w2.weight": "model-00039-of-00059.safetensors",
+    "model.layers.36.block_sparse_moe.experts.6.w3.weight": "model-00039-of-00059.safetensors",
+    "model.layers.36.block_sparse_moe.experts.7.w1.weight": "model-00039-of-00059.safetensors",
+    "model.layers.36.block_sparse_moe.experts.7.w2.weight": "model-00039-of-00059.safetensors",
+    "model.layers.36.block_sparse_moe.experts.7.w3.weight": "model-00039-of-00059.safetensors",
+    "model.layers.36.block_sparse_moe.gate.weight": "model-00038-of-00059.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00039-of-00059.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00039-of-00059.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00038-of-00059.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00038-of-00059.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00038-of-00059.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00038-of-00059.safetensors",
+    "model.layers.37.block_sparse_moe.experts.0.w1.weight": "model-00039-of-00059.safetensors",
+    "model.layers.37.block_sparse_moe.experts.0.w2.weight": "model-00039-of-00059.safetensors",
+    "model.layers.37.block_sparse_moe.experts.0.w3.weight": "model-00039-of-00059.safetensors",
+    "model.layers.37.block_sparse_moe.experts.1.w1.weight": "model-00039-of-00059.safetensors",
+    "model.layers.37.block_sparse_moe.experts.1.w2.weight": "model-00039-of-00059.safetensors",
+    "model.layers.37.block_sparse_moe.experts.1.w3.weight": "model-00039-of-00059.safetensors",
+    "model.layers.37.block_sparse_moe.experts.2.w1.weight": "model-00039-of-00059.safetensors",
+    "model.layers.37.block_sparse_moe.experts.2.w2.weight": "model-00039-of-00059.safetensors",
+    "model.layers.37.block_sparse_moe.experts.2.w3.weight": "model-00039-of-00059.safetensors",
+    "model.layers.37.block_sparse_moe.experts.3.w1.weight": "model-00039-of-00059.safetensors",
+    "model.layers.37.block_sparse_moe.experts.3.w2.weight": "model-00039-of-00059.safetensors",
+    "model.layers.37.block_sparse_moe.experts.3.w3.weight": "model-00039-of-00059.safetensors",
+    "model.layers.37.block_sparse_moe.experts.4.w1.weight": "model-00040-of-00059.safetensors",
+    "model.layers.37.block_sparse_moe.experts.4.w2.weight": "model-00040-of-00059.safetensors",
+    "model.layers.37.block_sparse_moe.experts.4.w3.weight": "model-00040-of-00059.safetensors",
+    "model.layers.37.block_sparse_moe.experts.5.w1.weight": "model-00040-of-00059.safetensors",
+    "model.layers.37.block_sparse_moe.experts.5.w2.weight": "model-00040-of-00059.safetensors",
+    "model.layers.37.block_sparse_moe.experts.5.w3.weight": "model-00040-of-00059.safetensors",
+    "model.layers.37.block_sparse_moe.experts.6.w1.weight": "model-00040-of-00059.safetensors",
+    "model.layers.37.block_sparse_moe.experts.6.w2.weight": "model-00040-of-00059.safetensors",
+    "model.layers.37.block_sparse_moe.experts.6.w3.weight": "model-00040-of-00059.safetensors",
+    "model.layers.37.block_sparse_moe.experts.7.w1.weight": "model-00040-of-00059.safetensors",
+    "model.layers.37.block_sparse_moe.experts.7.w2.weight": "model-00040-of-00059.safetensors",
+    "model.layers.37.block_sparse_moe.experts.7.w3.weight": "model-00040-of-00059.safetensors",
+    "model.layers.37.block_sparse_moe.gate.weight": "model-00039-of-00059.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00040-of-00059.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00040-of-00059.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00039-of-00059.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00039-of-00059.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00039-of-00059.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00039-of-00059.safetensors",
+    "model.layers.38.block_sparse_moe.experts.0.w1.weight": "model-00040-of-00059.safetensors",
+    "model.layers.38.block_sparse_moe.experts.0.w2.weight": "model-00040-of-00059.safetensors",
+    "model.layers.38.block_sparse_moe.experts.0.w3.weight": "model-00040-of-00059.safetensors",
+    "model.layers.38.block_sparse_moe.experts.1.w1.weight": "model-00040-of-00059.safetensors",
+    "model.layers.38.block_sparse_moe.experts.1.w2.weight": "model-00040-of-00059.safetensors",
+    "model.layers.38.block_sparse_moe.experts.1.w3.weight": "model-00040-of-00059.safetensors",
+    "model.layers.38.block_sparse_moe.experts.2.w1.weight": "model-00040-of-00059.safetensors",
+    "model.layers.38.block_sparse_moe.experts.2.w2.weight": "model-00040-of-00059.safetensors",
+    "model.layers.38.block_sparse_moe.experts.2.w3.weight": "model-00040-of-00059.safetensors",
+    "model.layers.38.block_sparse_moe.experts.3.w1.weight": "model-00040-of-00059.safetensors",
+    "model.layers.38.block_sparse_moe.experts.3.w2.weight": "model-00040-of-00059.safetensors",
+    "model.layers.38.block_sparse_moe.experts.3.w3.weight": "model-00041-of-00059.safetensors",
+    "model.layers.38.block_sparse_moe.experts.4.w1.weight": "model-00041-of-00059.safetensors",
+    "model.layers.38.block_sparse_moe.experts.4.w2.weight": "model-00041-of-00059.safetensors",
+    "model.layers.38.block_sparse_moe.experts.4.w3.weight": "model-00041-of-00059.safetensors",
+    "model.layers.38.block_sparse_moe.experts.5.w1.weight": "model-00041-of-00059.safetensors",
+    "model.layers.38.block_sparse_moe.experts.5.w2.weight": "model-00041-of-00059.safetensors",
+    "model.layers.38.block_sparse_moe.experts.5.w3.weight": "model-00041-of-00059.safetensors",
+    "model.layers.38.block_sparse_moe.experts.6.w1.weight": "model-00041-of-00059.safetensors",
+    "model.layers.38.block_sparse_moe.experts.6.w2.weight": "model-00041-of-00059.safetensors",
+    "model.layers.38.block_sparse_moe.experts.6.w3.weight": "model-00041-of-00059.safetensors",
+    "model.layers.38.block_sparse_moe.experts.7.w1.weight": "model-00041-of-00059.safetensors",
+    "model.layers.38.block_sparse_moe.experts.7.w2.weight": "model-00041-of-00059.safetensors",
+    "model.layers.38.block_sparse_moe.experts.7.w3.weight": "model-00041-of-00059.safetensors",
+    "model.layers.38.block_sparse_moe.gate.weight": "model-00040-of-00059.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00041-of-00059.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00041-of-00059.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00040-of-00059.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00040-of-00059.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00040-of-00059.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00040-of-00059.safetensors",
+    "model.layers.39.block_sparse_moe.experts.0.w1.weight": "model-00041-of-00059.safetensors",
+    "model.layers.39.block_sparse_moe.experts.0.w2.weight": "model-00041-of-00059.safetensors",
+    "model.layers.39.block_sparse_moe.experts.0.w3.weight": "model-00041-of-00059.safetensors",
+    "model.layers.39.block_sparse_moe.experts.1.w1.weight": "model-00041-of-00059.safetensors",
+    "model.layers.39.block_sparse_moe.experts.1.w2.weight": "model-00041-of-00059.safetensors",
+    "model.layers.39.block_sparse_moe.experts.1.w3.weight": "model-00041-of-00059.safetensors",
+    "model.layers.39.block_sparse_moe.experts.2.w1.weight": "model-00041-of-00059.safetensors",
+    "model.layers.39.block_sparse_moe.experts.2.w2.weight": "model-00041-of-00059.safetensors",
+    "model.layers.39.block_sparse_moe.experts.2.w3.weight": "model-00041-of-00059.safetensors",
+    "model.layers.39.block_sparse_moe.experts.3.w1.weight": "model-00041-of-00059.safetensors",
+    "model.layers.39.block_sparse_moe.experts.3.w2.weight": "model-00042-of-00059.safetensors",
+    "model.layers.39.block_sparse_moe.experts.3.w3.weight": "model-00042-of-00059.safetensors",
+    "model.layers.39.block_sparse_moe.experts.4.w1.weight": "model-00042-of-00059.safetensors",
+    "model.layers.39.block_sparse_moe.experts.4.w2.weight": "model-00042-of-00059.safetensors",
+    "model.layers.39.block_sparse_moe.experts.4.w3.weight": "model-00042-of-00059.safetensors",
+    "model.layers.39.block_sparse_moe.experts.5.w1.weight": "model-00042-of-00059.safetensors",
+    "model.layers.39.block_sparse_moe.experts.5.w2.weight": "model-00042-of-00059.safetensors",
+    "model.layers.39.block_sparse_moe.experts.5.w3.weight": "model-00042-of-00059.safetensors",
+    "model.layers.39.block_sparse_moe.experts.6.w1.weight": "model-00042-of-00059.safetensors",
+    "model.layers.39.block_sparse_moe.experts.6.w2.weight": "model-00042-of-00059.safetensors",
+    "model.layers.39.block_sparse_moe.experts.6.w3.weight": "model-00042-of-00059.safetensors",
+    "model.layers.39.block_sparse_moe.experts.7.w1.weight": "model-00042-of-00059.safetensors",
+    "model.layers.39.block_sparse_moe.experts.7.w2.weight": "model-00042-of-00059.safetensors",
+    "model.layers.39.block_sparse_moe.experts.7.w3.weight": "model-00042-of-00059.safetensors",
+    "model.layers.39.block_sparse_moe.gate.weight": "model-00041-of-00059.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00042-of-00059.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00042-of-00059.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00041-of-00059.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00041-of-00059.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00041-of-00059.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00041-of-00059.safetensors",
+    "model.layers.4.block_sparse_moe.experts.0.w1.weight": "model-00005-of-00059.safetensors",
+    "model.layers.4.block_sparse_moe.experts.0.w2.weight": "model-00005-of-00059.safetensors",
+    "model.layers.4.block_sparse_moe.experts.0.w3.weight": "model-00005-of-00059.safetensors",
+    "model.layers.4.block_sparse_moe.experts.1.w1.weight": "model-00005-of-00059.safetensors",
+    "model.layers.4.block_sparse_moe.experts.1.w2.weight": "model-00005-of-00059.safetensors",
+    "model.layers.4.block_sparse_moe.experts.1.w3.weight": "model-00005-of-00059.safetensors",
+    "model.layers.4.block_sparse_moe.experts.2.w1.weight": "model-00005-of-00059.safetensors",
+    "model.layers.4.block_sparse_moe.experts.2.w2.weight": "model-00005-of-00059.safetensors",
+    "model.layers.4.block_sparse_moe.experts.2.w3.weight": "model-00005-of-00059.safetensors",
+    "model.layers.4.block_sparse_moe.experts.3.w1.weight": "model-00005-of-00059.safetensors",
+    "model.layers.4.block_sparse_moe.experts.3.w2.weight": "model-00005-of-00059.safetensors",
+    "model.layers.4.block_sparse_moe.experts.3.w3.weight": "model-00005-of-00059.safetensors",
+    "model.layers.4.block_sparse_moe.experts.4.w1.weight": "model-00005-of-00059.safetensors",
+    "model.layers.4.block_sparse_moe.experts.4.w2.weight": "model-00005-of-00059.safetensors",
+    "model.layers.4.block_sparse_moe.experts.4.w3.weight": "model-00005-of-00059.safetensors",
+    "model.layers.4.block_sparse_moe.experts.5.w1.weight": "model-00005-of-00059.safetensors",
+    "model.layers.4.block_sparse_moe.experts.5.w2.weight": "model-00005-of-00059.safetensors",
+    "model.layers.4.block_sparse_moe.experts.5.w3.weight": "model-00005-of-00059.safetensors",
+    "model.layers.4.block_sparse_moe.experts.6.w1.weight": "model-00006-of-00059.safetensors",
+    "model.layers.4.block_sparse_moe.experts.6.w2.weight": "model-00006-of-00059.safetensors",
+    "model.layers.4.block_sparse_moe.experts.6.w3.weight": "model-00006-of-00059.safetensors",
+    "model.layers.4.block_sparse_moe.experts.7.w1.weight": "model-00006-of-00059.safetensors",
+    "model.layers.4.block_sparse_moe.experts.7.w2.weight": "model-00006-of-00059.safetensors",
+    "model.layers.4.block_sparse_moe.experts.7.w3.weight": "model-00006-of-00059.safetensors",
+    "model.layers.4.block_sparse_moe.gate.weight": "model-00005-of-00059.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00006-of-00059.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00006-of-00059.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00005-of-00059.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00005-of-00059.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00005-of-00059.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00005-of-00059.safetensors",
+    "model.layers.40.block_sparse_moe.experts.0.w1.weight": "model-00042-of-00059.safetensors",
+    "model.layers.40.block_sparse_moe.experts.0.w2.weight": "model-00042-of-00059.safetensors",
+    "model.layers.40.block_sparse_moe.experts.0.w3.weight": "model-00042-of-00059.safetensors",
+    "model.layers.40.block_sparse_moe.experts.1.w1.weight": "model-00042-of-00059.safetensors",
+    "model.layers.40.block_sparse_moe.experts.1.w2.weight": "model-00042-of-00059.safetensors",
+    "model.layers.40.block_sparse_moe.experts.1.w3.weight": "model-00042-of-00059.safetensors",
+    "model.layers.40.block_sparse_moe.experts.2.w1.weight": "model-00042-of-00059.safetensors",
+    "model.layers.40.block_sparse_moe.experts.2.w2.weight": "model-00042-of-00059.safetensors",
+    "model.layers.40.block_sparse_moe.experts.2.w3.weight": "model-00042-of-00059.safetensors",
+    "model.layers.40.block_sparse_moe.experts.3.w1.weight": "model-00043-of-00059.safetensors",
+    "model.layers.40.block_sparse_moe.experts.3.w2.weight": "model-00043-of-00059.safetensors",
+    "model.layers.40.block_sparse_moe.experts.3.w3.weight": "model-00043-of-00059.safetensors",
+    "model.layers.40.block_sparse_moe.experts.4.w1.weight": "model-00043-of-00059.safetensors",
+    "model.layers.40.block_sparse_moe.experts.4.w2.weight": "model-00043-of-00059.safetensors",
+    "model.layers.40.block_sparse_moe.experts.4.w3.weight": "model-00043-of-00059.safetensors",
+    "model.layers.40.block_sparse_moe.experts.5.w1.weight": "model-00043-of-00059.safetensors",
+    "model.layers.40.block_sparse_moe.experts.5.w2.weight": "model-00043-of-00059.safetensors",
+    "model.layers.40.block_sparse_moe.experts.5.w3.weight": "model-00043-of-00059.safetensors",
+    "model.layers.40.block_sparse_moe.experts.6.w1.weight": "model-00043-of-00059.safetensors",
+    "model.layers.40.block_sparse_moe.experts.6.w2.weight": "model-00043-of-00059.safetensors",
+    "model.layers.40.block_sparse_moe.experts.6.w3.weight": "model-00043-of-00059.safetensors",
+    "model.layers.40.block_sparse_moe.experts.7.w1.weight": "model-00043-of-00059.safetensors",
+    "model.layers.40.block_sparse_moe.experts.7.w2.weight": "model-00043-of-00059.safetensors",
+    "model.layers.40.block_sparse_moe.experts.7.w3.weight": "model-00043-of-00059.safetensors",
+    "model.layers.40.block_sparse_moe.gate.weight": "model-00042-of-00059.safetensors",
+    "model.layers.40.input_layernorm.weight": "model-00043-of-00059.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00043-of-00059.safetensors",
+    "model.layers.40.self_attn.k_proj.weight": "model-00042-of-00059.safetensors",
+    "model.layers.40.self_attn.o_proj.weight": "model-00042-of-00059.safetensors",
+    "model.layers.40.self_attn.q_proj.weight": "model-00042-of-00059.safetensors",
+    "model.layers.40.self_attn.v_proj.weight": "model-00042-of-00059.safetensors",
+    "model.layers.41.block_sparse_moe.experts.0.w1.weight": "model-00043-of-00059.safetensors",
+    "model.layers.41.block_sparse_moe.experts.0.w2.weight": "model-00043-of-00059.safetensors",
+    "model.layers.41.block_sparse_moe.experts.0.w3.weight": "model-00043-of-00059.safetensors",
+    "model.layers.41.block_sparse_moe.experts.1.w1.weight": "model-00043-of-00059.safetensors",
+    "model.layers.41.block_sparse_moe.experts.1.w2.weight": "model-00043-of-00059.safetensors",
+    "model.layers.41.block_sparse_moe.experts.1.w3.weight": "model-00043-of-00059.safetensors",
+    "model.layers.41.block_sparse_moe.experts.2.w1.weight": "model-00043-of-00059.safetensors",
+    "model.layers.41.block_sparse_moe.experts.2.w2.weight": "model-00043-of-00059.safetensors",
+    "model.layers.41.block_sparse_moe.experts.2.w3.weight": "model-00044-of-00059.safetensors",
+    "model.layers.41.block_sparse_moe.experts.3.w1.weight": "model-00044-of-00059.safetensors",
+    "model.layers.41.block_sparse_moe.experts.3.w2.weight": "model-00044-of-00059.safetensors",
+    "model.layers.41.block_sparse_moe.experts.3.w3.weight": "model-00044-of-00059.safetensors",
+    "model.layers.41.block_sparse_moe.experts.4.w1.weight": "model-00044-of-00059.safetensors",
+    "model.layers.41.block_sparse_moe.experts.4.w2.weight": "model-00044-of-00059.safetensors",
+    "model.layers.41.block_sparse_moe.experts.4.w3.weight": "model-00044-of-00059.safetensors",
+    "model.layers.41.block_sparse_moe.experts.5.w1.weight": "model-00044-of-00059.safetensors",
+    "model.layers.41.block_sparse_moe.experts.5.w2.weight": "model-00044-of-00059.safetensors",
+    "model.layers.41.block_sparse_moe.experts.5.w3.weight": "model-00044-of-00059.safetensors",
+    "model.layers.41.block_sparse_moe.experts.6.w1.weight": "model-00044-of-00059.safetensors",
+    "model.layers.41.block_sparse_moe.experts.6.w2.weight": "model-00044-of-00059.safetensors",
+    "model.layers.41.block_sparse_moe.experts.6.w3.weight": "model-00044-of-00059.safetensors",
+    "model.layers.41.block_sparse_moe.experts.7.w1.weight": "model-00044-of-00059.safetensors",
+    "model.layers.41.block_sparse_moe.experts.7.w2.weight": "model-00044-of-00059.safetensors",
+    "model.layers.41.block_sparse_moe.experts.7.w3.weight": "model-00044-of-00059.safetensors",
+    "model.layers.41.block_sparse_moe.gate.weight": "model-00043-of-00059.safetensors",
+    "model.layers.41.input_layernorm.weight": "model-00044-of-00059.safetensors",
+    "model.layers.41.post_attention_layernorm.weight": "model-00044-of-00059.safetensors",
+    "model.layers.41.self_attn.k_proj.weight": "model-00043-of-00059.safetensors",
+    "model.layers.41.self_attn.o_proj.weight": "model-00043-of-00059.safetensors",
+    "model.layers.41.self_attn.q_proj.weight": "model-00043-of-00059.safetensors",
+    "model.layers.41.self_attn.v_proj.weight": "model-00043-of-00059.safetensors",
+    "model.layers.42.block_sparse_moe.experts.0.w1.weight": "model-00044-of-00059.safetensors",
+    "model.layers.42.block_sparse_moe.experts.0.w2.weight": "model-00044-of-00059.safetensors",
+    "model.layers.42.block_sparse_moe.experts.0.w3.weight": "model-00044-of-00059.safetensors",
+    "model.layers.42.block_sparse_moe.experts.1.w1.weight": "model-00044-of-00059.safetensors",
+    "model.layers.42.block_sparse_moe.experts.1.w2.weight": "model-00044-of-00059.safetensors",
+    "model.layers.42.block_sparse_moe.experts.1.w3.weight": "model-00044-of-00059.safetensors",
+    "model.layers.42.block_sparse_moe.experts.2.w1.weight": "model-00044-of-00059.safetensors",
+    "model.layers.42.block_sparse_moe.experts.2.w2.weight": "model-00045-of-00059.safetensors",
+    "model.layers.42.block_sparse_moe.experts.2.w3.weight": "model-00045-of-00059.safetensors",
+    "model.layers.42.block_sparse_moe.experts.3.w1.weight": "model-00045-of-00059.safetensors",
+    "model.layers.42.block_sparse_moe.experts.3.w2.weight": "model-00045-of-00059.safetensors",
+    "model.layers.42.block_sparse_moe.experts.3.w3.weight": "model-00045-of-00059.safetensors",
+    "model.layers.42.block_sparse_moe.experts.4.w1.weight": "model-00045-of-00059.safetensors",
+    "model.layers.42.block_sparse_moe.experts.4.w2.weight": "model-00045-of-00059.safetensors",
+    "model.layers.42.block_sparse_moe.experts.4.w3.weight": "model-00045-of-00059.safetensors",
+    "model.layers.42.block_sparse_moe.experts.5.w1.weight": "model-00045-of-00059.safetensors",
+    "model.layers.42.block_sparse_moe.experts.5.w2.weight": "model-00045-of-00059.safetensors",
+    "model.layers.42.block_sparse_moe.experts.5.w3.weight": "model-00045-of-00059.safetensors",
+    "model.layers.42.block_sparse_moe.experts.6.w1.weight": "model-00045-of-00059.safetensors",
+    "model.layers.42.block_sparse_moe.experts.6.w2.weight": "model-00045-of-00059.safetensors",
+    "model.layers.42.block_sparse_moe.experts.6.w3.weight": "model-00045-of-00059.safetensors",
+    "model.layers.42.block_sparse_moe.experts.7.w1.weight": "model-00045-of-00059.safetensors",
+    "model.layers.42.block_sparse_moe.experts.7.w2.weight": "model-00045-of-00059.safetensors",
+    "model.layers.42.block_sparse_moe.experts.7.w3.weight": "model-00045-of-00059.safetensors",
+    "model.layers.42.block_sparse_moe.gate.weight": "model-00044-of-00059.safetensors",
+    "model.layers.42.input_layernorm.weight": "model-00045-of-00059.safetensors",
+    "model.layers.42.post_attention_layernorm.weight": "model-00045-of-00059.safetensors",
+    "model.layers.42.self_attn.k_proj.weight": "model-00044-of-00059.safetensors",
+    "model.layers.42.self_attn.o_proj.weight": "model-00044-of-00059.safetensors",
+    "model.layers.42.self_attn.q_proj.weight": "model-00044-of-00059.safetensors",
+    "model.layers.42.self_attn.v_proj.weight": "model-00044-of-00059.safetensors",
+    "model.layers.43.block_sparse_moe.experts.0.w1.weight": "model-00045-of-00059.safetensors",
+    "model.layers.43.block_sparse_moe.experts.0.w2.weight": "model-00045-of-00059.safetensors",
+    "model.layers.43.block_sparse_moe.experts.0.w3.weight": "model-00045-of-00059.safetensors",
+    "model.layers.43.block_sparse_moe.experts.1.w1.weight": "model-00045-of-00059.safetensors",
+    "model.layers.43.block_sparse_moe.experts.1.w2.weight": "model-00045-of-00059.safetensors",
+    "model.layers.43.block_sparse_moe.experts.1.w3.weight": "model-00045-of-00059.safetensors",
+    "model.layers.43.block_sparse_moe.experts.2.w1.weight": "model-00046-of-00059.safetensors",
+    "model.layers.43.block_sparse_moe.experts.2.w2.weight": "model-00046-of-00059.safetensors",
+    "model.layers.43.block_sparse_moe.experts.2.w3.weight": "model-00046-of-00059.safetensors",
+    "model.layers.43.block_sparse_moe.experts.3.w1.weight": "model-00046-of-00059.safetensors",
+    "model.layers.43.block_sparse_moe.experts.3.w2.weight": "model-00046-of-00059.safetensors",
+    "model.layers.43.block_sparse_moe.experts.3.w3.weight": "model-00046-of-00059.safetensors",
+    "model.layers.43.block_sparse_moe.experts.4.w1.weight": "model-00046-of-00059.safetensors",
+    "model.layers.43.block_sparse_moe.experts.4.w2.weight": "model-00046-of-00059.safetensors",
+    "model.layers.43.block_sparse_moe.experts.4.w3.weight": "model-00046-of-00059.safetensors",
+    "model.layers.43.block_sparse_moe.experts.5.w1.weight": "model-00046-of-00059.safetensors",
+    "model.layers.43.block_sparse_moe.experts.5.w2.weight": "model-00046-of-00059.safetensors",
+    "model.layers.43.block_sparse_moe.experts.5.w3.weight": "model-00046-of-00059.safetensors",
+    "model.layers.43.block_sparse_moe.experts.6.w1.weight": "model-00046-of-00059.safetensors",
+    "model.layers.43.block_sparse_moe.experts.6.w2.weight": "model-00046-of-00059.safetensors",
+    "model.layers.43.block_sparse_moe.experts.6.w3.weight": "model-00046-of-00059.safetensors",
+    "model.layers.43.block_sparse_moe.experts.7.w1.weight": "model-00046-of-00059.safetensors",
+    "model.layers.43.block_sparse_moe.experts.7.w2.weight": "model-00046-of-00059.safetensors",
+    "model.layers.43.block_sparse_moe.experts.7.w3.weight": "model-00046-of-00059.safetensors",
+    "model.layers.43.block_sparse_moe.gate.weight": "model-00045-of-00059.safetensors",
+    "model.layers.43.input_layernorm.weight": "model-00046-of-00059.safetensors",
+    "model.layers.43.post_attention_layernorm.weight": "model-00046-of-00059.safetensors",
+    "model.layers.43.self_attn.k_proj.weight": "model-00045-of-00059.safetensors",
+    "model.layers.43.self_attn.o_proj.weight": "model-00045-of-00059.safetensors",
+    "model.layers.43.self_attn.q_proj.weight": "model-00045-of-00059.safetensors",
+    "model.layers.43.self_attn.v_proj.weight": "model-00045-of-00059.safetensors",
+    "model.layers.44.block_sparse_moe.experts.0.w1.weight": "model-00046-of-00059.safetensors",
+    "model.layers.44.block_sparse_moe.experts.0.w2.weight": "model-00046-of-00059.safetensors",
+    "model.layers.44.block_sparse_moe.experts.0.w3.weight": "model-00046-of-00059.safetensors",
+    "model.layers.44.block_sparse_moe.experts.1.w1.weight": "model-00046-of-00059.safetensors",
+    "model.layers.44.block_sparse_moe.experts.1.w2.weight": "model-00046-of-00059.safetensors",
+    "model.layers.44.block_sparse_moe.experts.1.w3.weight": "model-00047-of-00059.safetensors",
+    "model.layers.44.block_sparse_moe.experts.2.w1.weight": "model-00047-of-00059.safetensors",
+    "model.layers.44.block_sparse_moe.experts.2.w2.weight": "model-00047-of-00059.safetensors",
+    "model.layers.44.block_sparse_moe.experts.2.w3.weight": "model-00047-of-00059.safetensors",
+    "model.layers.44.block_sparse_moe.experts.3.w1.weight": "model-00047-of-00059.safetensors",
+    "model.layers.44.block_sparse_moe.experts.3.w2.weight": "model-00047-of-00059.safetensors",
+    "model.layers.44.block_sparse_moe.experts.3.w3.weight": "model-00047-of-00059.safetensors",
+    "model.layers.44.block_sparse_moe.experts.4.w1.weight": "model-00047-of-00059.safetensors",
+    "model.layers.44.block_sparse_moe.experts.4.w2.weight": "model-00047-of-00059.safetensors",
+    "model.layers.44.block_sparse_moe.experts.4.w3.weight": "model-00047-of-00059.safetensors",
+    "model.layers.44.block_sparse_moe.experts.5.w1.weight": "model-00047-of-00059.safetensors",
+    "model.layers.44.block_sparse_moe.experts.5.w2.weight": "model-00047-of-00059.safetensors",
+    "model.layers.44.block_sparse_moe.experts.5.w3.weight": "model-00047-of-00059.safetensors",
+    "model.layers.44.block_sparse_moe.experts.6.w1.weight": "model-00047-of-00059.safetensors",
+    "model.layers.44.block_sparse_moe.experts.6.w2.weight": "model-00047-of-00059.safetensors",
+    "model.layers.44.block_sparse_moe.experts.6.w3.weight": "model-00047-of-00059.safetensors",
+    "model.layers.44.block_sparse_moe.experts.7.w1.weight": "model-00047-of-00059.safetensors",
+    "model.layers.44.block_sparse_moe.experts.7.w2.weight": "model-00047-of-00059.safetensors",
+    "model.layers.44.block_sparse_moe.experts.7.w3.weight": "model-00047-of-00059.safetensors",
+    "model.layers.44.block_sparse_moe.gate.weight": "model-00046-of-00059.safetensors",
+    "model.layers.44.input_layernorm.weight": "model-00047-of-00059.safetensors",
+    "model.layers.44.post_attention_layernorm.weight": "model-00047-of-00059.safetensors",
+    "model.layers.44.self_attn.k_proj.weight": "model-00046-of-00059.safetensors",
+    "model.layers.44.self_attn.o_proj.weight": "model-00046-of-00059.safetensors",
+    "model.layers.44.self_attn.q_proj.weight": "model-00046-of-00059.safetensors",
+    "model.layers.44.self_attn.v_proj.weight": "model-00046-of-00059.safetensors",
+    "model.layers.45.block_sparse_moe.experts.0.w1.weight": "model-00047-of-00059.safetensors",
+    "model.layers.45.block_sparse_moe.experts.0.w2.weight": "model-00047-of-00059.safetensors",
+    "model.layers.45.block_sparse_moe.experts.0.w3.weight": "model-00047-of-00059.safetensors",
+    "model.layers.45.block_sparse_moe.experts.1.w1.weight": "model-00047-of-00059.safetensors",
+    "model.layers.45.block_sparse_moe.experts.1.w2.weight": "model-00048-of-00059.safetensors",
+    "model.layers.45.block_sparse_moe.experts.1.w3.weight": "model-00048-of-00059.safetensors",
+    "model.layers.45.block_sparse_moe.experts.2.w1.weight": "model-00048-of-00059.safetensors",
+    "model.layers.45.block_sparse_moe.experts.2.w2.weight": "model-00048-of-00059.safetensors",
+    "model.layers.45.block_sparse_moe.experts.2.w3.weight": "model-00048-of-00059.safetensors",
+    "model.layers.45.block_sparse_moe.experts.3.w1.weight": "model-00048-of-00059.safetensors",
+    "model.layers.45.block_sparse_moe.experts.3.w2.weight": "model-00048-of-00059.safetensors",
+    "model.layers.45.block_sparse_moe.experts.3.w3.weight": "model-00048-of-00059.safetensors",
+    "model.layers.45.block_sparse_moe.experts.4.w1.weight": "model-00048-of-00059.safetensors",
+    "model.layers.45.block_sparse_moe.experts.4.w2.weight": "model-00048-of-00059.safetensors",
+    "model.layers.45.block_sparse_moe.experts.4.w3.weight": "model-00048-of-00059.safetensors",
+    "model.layers.45.block_sparse_moe.experts.5.w1.weight": "model-00048-of-00059.safetensors",
+    "model.layers.45.block_sparse_moe.experts.5.w2.weight": "model-00048-of-00059.safetensors",
+    "model.layers.45.block_sparse_moe.experts.5.w3.weight": "model-00048-of-00059.safetensors",
+    "model.layers.45.block_sparse_moe.experts.6.w1.weight": "model-00048-of-00059.safetensors",
+    "model.layers.45.block_sparse_moe.experts.6.w2.weight": "model-00048-of-00059.safetensors",
+    "model.layers.45.block_sparse_moe.experts.6.w3.weight": "model-00048-of-00059.safetensors",
+    "model.layers.45.block_sparse_moe.experts.7.w1.weight": "model-00048-of-00059.safetensors",
+    "model.layers.45.block_sparse_moe.experts.7.w2.weight": "model-00048-of-00059.safetensors",
+    "model.layers.45.block_sparse_moe.experts.7.w3.weight": "model-00048-of-00059.safetensors",
+    "model.layers.45.block_sparse_moe.gate.weight": "model-00047-of-00059.safetensors",
+    "model.layers.45.input_layernorm.weight": "model-00048-of-00059.safetensors",
+    "model.layers.45.post_attention_layernorm.weight": "model-00048-of-00059.safetensors",
+    "model.layers.45.self_attn.k_proj.weight": "model-00047-of-00059.safetensors",
+    "model.layers.45.self_attn.o_proj.weight": "model-00047-of-00059.safetensors",
+    "model.layers.45.self_attn.q_proj.weight": "model-00047-of-00059.safetensors",
+    "model.layers.45.self_attn.v_proj.weight": "model-00047-of-00059.safetensors",
+    "model.layers.46.block_sparse_moe.experts.0.w1.weight": "model-00048-of-00059.safetensors",
+    "model.layers.46.block_sparse_moe.experts.0.w2.weight": "model-00048-of-00059.safetensors",
+    "model.layers.46.block_sparse_moe.experts.0.w3.weight": "model-00048-of-00059.safetensors",
+    "model.layers.46.block_sparse_moe.experts.1.w1.weight": "model-00049-of-00059.safetensors",
+    "model.layers.46.block_sparse_moe.experts.1.w2.weight": "model-00049-of-00059.safetensors",
+    "model.layers.46.block_sparse_moe.experts.1.w3.weight": "model-00049-of-00059.safetensors",
+    "model.layers.46.block_sparse_moe.experts.2.w1.weight": "model-00049-of-00059.safetensors",
+    "model.layers.46.block_sparse_moe.experts.2.w2.weight": "model-00049-of-00059.safetensors",
+    "model.layers.46.block_sparse_moe.experts.2.w3.weight": "model-00049-of-00059.safetensors",
+    "model.layers.46.block_sparse_moe.experts.3.w1.weight": "model-00049-of-00059.safetensors",
+    "model.layers.46.block_sparse_moe.experts.3.w2.weight": "model-00049-of-00059.safetensors",
+    "model.layers.46.block_sparse_moe.experts.3.w3.weight": "model-00049-of-00059.safetensors",
+    "model.layers.46.block_sparse_moe.experts.4.w1.weight": "model-00049-of-00059.safetensors",
+    "model.layers.46.block_sparse_moe.experts.4.w2.weight": "model-00049-of-00059.safetensors",
+    "model.layers.46.block_sparse_moe.experts.4.w3.weight": "model-00049-of-00059.safetensors",
+    "model.layers.46.block_sparse_moe.experts.5.w1.weight": "model-00049-of-00059.safetensors",
+    "model.layers.46.block_sparse_moe.experts.5.w2.weight": "model-00049-of-00059.safetensors",
+    "model.layers.46.block_sparse_moe.experts.5.w3.weight": "model-00049-of-00059.safetensors",
+    "model.layers.46.block_sparse_moe.experts.6.w1.weight": "model-00049-of-00059.safetensors",
+    "model.layers.46.block_sparse_moe.experts.6.w2.weight": "model-00049-of-00059.safetensors",
+    "model.layers.46.block_sparse_moe.experts.6.w3.weight": "model-00049-of-00059.safetensors",
+    "model.layers.46.block_sparse_moe.experts.7.w1.weight": "model-00049-of-00059.safetensors",
+    "model.layers.46.block_sparse_moe.experts.7.w2.weight": "model-00049-of-00059.safetensors",
+    "model.layers.46.block_sparse_moe.experts.7.w3.weight": "model-00049-of-00059.safetensors",
+    "model.layers.46.block_sparse_moe.gate.weight": "model-00048-of-00059.safetensors",
+    "model.layers.46.input_layernorm.weight": "model-00049-of-00059.safetensors",
+    "model.layers.46.post_attention_layernorm.weight": "model-00049-of-00059.safetensors",
+    "model.layers.46.self_attn.k_proj.weight": "model-00048-of-00059.safetensors",
+    "model.layers.46.self_attn.o_proj.weight": "model-00048-of-00059.safetensors",
+    "model.layers.46.self_attn.q_proj.weight": "model-00048-of-00059.safetensors",
+    "model.layers.46.self_attn.v_proj.weight": "model-00048-of-00059.safetensors",
+    "model.layers.47.block_sparse_moe.experts.0.w1.weight": "model-00049-of-00059.safetensors",
+    "model.layers.47.block_sparse_moe.experts.0.w2.weight": "model-00049-of-00059.safetensors",
+    "model.layers.47.block_sparse_moe.experts.0.w3.weight": "model-00050-of-00059.safetensors",
+    "model.layers.47.block_sparse_moe.experts.1.w1.weight": "model-00050-of-00059.safetensors",
+    "model.layers.47.block_sparse_moe.experts.1.w2.weight": "model-00050-of-00059.safetensors",
+    "model.layers.47.block_sparse_moe.experts.1.w3.weight": "model-00050-of-00059.safetensors",
+    "model.layers.47.block_sparse_moe.experts.2.w1.weight": "model-00050-of-00059.safetensors",
+    "model.layers.47.block_sparse_moe.experts.2.w2.weight": "model-00050-of-00059.safetensors",
+    "model.layers.47.block_sparse_moe.experts.2.w3.weight": "model-00050-of-00059.safetensors",
+    "model.layers.47.block_sparse_moe.experts.3.w1.weight": "model-00050-of-00059.safetensors",
+    "model.layers.47.block_sparse_moe.experts.3.w2.weight": "model-00050-of-00059.safetensors",
+    "model.layers.47.block_sparse_moe.experts.3.w3.weight": "model-00050-of-00059.safetensors",
+    "model.layers.47.block_sparse_moe.experts.4.w1.weight": "model-00050-of-00059.safetensors",
+    "model.layers.47.block_sparse_moe.experts.4.w2.weight": "model-00050-of-00059.safetensors",
+    "model.layers.47.block_sparse_moe.experts.4.w3.weight": "model-00050-of-00059.safetensors",
+    "model.layers.47.block_sparse_moe.experts.5.w1.weight": "model-00050-of-00059.safetensors",
+    "model.layers.47.block_sparse_moe.experts.5.w2.weight": "model-00050-of-00059.safetensors",
+    "model.layers.47.block_sparse_moe.experts.5.w3.weight": "model-00050-of-00059.safetensors",
+    "model.layers.47.block_sparse_moe.experts.6.w1.weight": "model-00050-of-00059.safetensors",
+    "model.layers.47.block_sparse_moe.experts.6.w2.weight": "model-00050-of-00059.safetensors",
+    "model.layers.47.block_sparse_moe.experts.6.w3.weight": "model-00050-of-00059.safetensors",
+    "model.layers.47.block_sparse_moe.experts.7.w1.weight": "model-00050-of-00059.safetensors",
+    "model.layers.47.block_sparse_moe.experts.7.w2.weight": "model-00050-of-00059.safetensors",
+    "model.layers.47.block_sparse_moe.experts.7.w3.weight": "model-00050-of-00059.safetensors",
+    "model.layers.47.block_sparse_moe.gate.weight": "model-00049-of-00059.safetensors",
+    "model.layers.47.input_layernorm.weight": "model-00050-of-00059.safetensors",
+    "model.layers.47.post_attention_layernorm.weight": "model-00050-of-00059.safetensors",
+    "model.layers.47.self_attn.k_proj.weight": "model-00049-of-00059.safetensors",
+    "model.layers.47.self_attn.o_proj.weight": "model-00049-of-00059.safetensors",
+    "model.layers.47.self_attn.q_proj.weight": "model-00049-of-00059.safetensors",
+    "model.layers.47.self_attn.v_proj.weight": "model-00049-of-00059.safetensors",
+    "model.layers.48.block_sparse_moe.experts.0.w1.weight": "model-00050-of-00059.safetensors",
+    "model.layers.48.block_sparse_moe.experts.0.w2.weight": "model-00051-of-00059.safetensors",
+    "model.layers.48.block_sparse_moe.experts.0.w3.weight": "model-00051-of-00059.safetensors",
+    "model.layers.48.block_sparse_moe.experts.1.w1.weight": "model-00051-of-00059.safetensors",
+    "model.layers.48.block_sparse_moe.experts.1.w2.weight": "model-00051-of-00059.safetensors",
+    "model.layers.48.block_sparse_moe.experts.1.w3.weight": "model-00051-of-00059.safetensors",
+    "model.layers.48.block_sparse_moe.experts.2.w1.weight": "model-00051-of-00059.safetensors",
+    "model.layers.48.block_sparse_moe.experts.2.w2.weight": "model-00051-of-00059.safetensors",
+    "model.layers.48.block_sparse_moe.experts.2.w3.weight": "model-00051-of-00059.safetensors",
+    "model.layers.48.block_sparse_moe.experts.3.w1.weight": "model-00051-of-00059.safetensors",
+    "model.layers.48.block_sparse_moe.experts.3.w2.weight": "model-00051-of-00059.safetensors",
+    "model.layers.48.block_sparse_moe.experts.3.w3.weight": "model-00051-of-00059.safetensors",
+    "model.layers.48.block_sparse_moe.experts.4.w1.weight": "model-00051-of-00059.safetensors",
+    "model.layers.48.block_sparse_moe.experts.4.w2.weight": "model-00051-of-00059.safetensors",
+    "model.layers.48.block_sparse_moe.experts.4.w3.weight": "model-00051-of-00059.safetensors",
+    "model.layers.48.block_sparse_moe.experts.5.w1.weight": "model-00051-of-00059.safetensors",
+    "model.layers.48.block_sparse_moe.experts.5.w2.weight": "model-00051-of-00059.safetensors",
+    "model.layers.48.block_sparse_moe.experts.5.w3.weight": "model-00051-of-00059.safetensors",
+    "model.layers.48.block_sparse_moe.experts.6.w1.weight": "model-00051-of-00059.safetensors",
+    "model.layers.48.block_sparse_moe.experts.6.w2.weight": "model-00051-of-00059.safetensors",
+    "model.layers.48.block_sparse_moe.experts.6.w3.weight": "model-00051-of-00059.safetensors",
+    "model.layers.48.block_sparse_moe.experts.7.w1.weight": "model-00051-of-00059.safetensors",
+    "model.layers.48.block_sparse_moe.experts.7.w2.weight": "model-00051-of-00059.safetensors",
+    "model.layers.48.block_sparse_moe.experts.7.w3.weight": "model-00051-of-00059.safetensors",
+    "model.layers.48.block_sparse_moe.gate.weight": "model-00050-of-00059.safetensors",
+    "model.layers.48.input_layernorm.weight": "model-00051-of-00059.safetensors",
+    "model.layers.48.post_attention_layernorm.weight": "model-00051-of-00059.safetensors",
+    "model.layers.48.self_attn.k_proj.weight": "model-00050-of-00059.safetensors",
+    "model.layers.48.self_attn.o_proj.weight": "model-00050-of-00059.safetensors",
+    "model.layers.48.self_attn.q_proj.weight": "model-00050-of-00059.safetensors",
+    "model.layers.48.self_attn.v_proj.weight": "model-00050-of-00059.safetensors",
+    "model.layers.49.block_sparse_moe.experts.0.w1.weight": "model-00052-of-00059.safetensors",
+    "model.layers.49.block_sparse_moe.experts.0.w2.weight": "model-00052-of-00059.safetensors",
+    "model.layers.49.block_sparse_moe.experts.0.w3.weight": "model-00052-of-00059.safetensors",
+    "model.layers.49.block_sparse_moe.experts.1.w1.weight": "model-00052-of-00059.safetensors",
+    "model.layers.49.block_sparse_moe.experts.1.w2.weight": "model-00052-of-00059.safetensors",
+    "model.layers.49.block_sparse_moe.experts.1.w3.weight": "model-00052-of-00059.safetensors",
+    "model.layers.49.block_sparse_moe.experts.2.w1.weight": "model-00052-of-00059.safetensors",
+    "model.layers.49.block_sparse_moe.experts.2.w2.weight": "model-00052-of-00059.safetensors",
+    "model.layers.49.block_sparse_moe.experts.2.w3.weight": "model-00052-of-00059.safetensors",
+    "model.layers.49.block_sparse_moe.experts.3.w1.weight": "model-00052-of-00059.safetensors",
+    "model.layers.49.block_sparse_moe.experts.3.w2.weight": "model-00052-of-00059.safetensors",
+    "model.layers.49.block_sparse_moe.experts.3.w3.weight": "model-00052-of-00059.safetensors",
+    "model.layers.49.block_sparse_moe.experts.4.w1.weight": "model-00052-of-00059.safetensors",
+    "model.layers.49.block_sparse_moe.experts.4.w2.weight": "model-00052-of-00059.safetensors",
+    "model.layers.49.block_sparse_moe.experts.4.w3.weight": "model-00052-of-00059.safetensors",
+    "model.layers.49.block_sparse_moe.experts.5.w1.weight": "model-00052-of-00059.safetensors",
+    "model.layers.49.block_sparse_moe.experts.5.w2.weight": "model-00052-of-00059.safetensors",
+    "model.layers.49.block_sparse_moe.experts.5.w3.weight": "model-00052-of-00059.safetensors",
+    "model.layers.49.block_sparse_moe.experts.6.w1.weight": "model-00052-of-00059.safetensors",
+    "model.layers.49.block_sparse_moe.experts.6.w2.weight": "model-00052-of-00059.safetensors",
+    "model.layers.49.block_sparse_moe.experts.6.w3.weight": "model-00052-of-00059.safetensors",
+    "model.layers.49.block_sparse_moe.experts.7.w1.weight": "model-00052-of-00059.safetensors",
+    "model.layers.49.block_sparse_moe.experts.7.w2.weight": "model-00052-of-00059.safetensors",
+    "model.layers.49.block_sparse_moe.experts.7.w3.weight": "model-00052-of-00059.safetensors",
+    "model.layers.49.block_sparse_moe.gate.weight": "model-00051-of-00059.safetensors",
+    "model.layers.49.input_layernorm.weight": "model-00052-of-00059.safetensors",
+    "model.layers.49.post_attention_layernorm.weight": "model-00052-of-00059.safetensors",
+    "model.layers.49.self_attn.k_proj.weight": "model-00051-of-00059.safetensors",
+    "model.layers.49.self_attn.o_proj.weight": "model-00051-of-00059.safetensors",
+    "model.layers.49.self_attn.q_proj.weight": "model-00051-of-00059.safetensors",
+    "model.layers.49.self_attn.v_proj.weight": "model-00051-of-00059.safetensors",
+    "model.layers.5.block_sparse_moe.experts.0.w1.weight": "model-00006-of-00059.safetensors",
+    "model.layers.5.block_sparse_moe.experts.0.w2.weight": "model-00006-of-00059.safetensors",
+    "model.layers.5.block_sparse_moe.experts.0.w3.weight": "model-00006-of-00059.safetensors",
+    "model.layers.5.block_sparse_moe.experts.1.w1.weight": "model-00006-of-00059.safetensors",
+    "model.layers.5.block_sparse_moe.experts.1.w2.weight": "model-00006-of-00059.safetensors",
+    "model.layers.5.block_sparse_moe.experts.1.w3.weight": "model-00006-of-00059.safetensors",
+    "model.layers.5.block_sparse_moe.experts.2.w1.weight": "model-00006-of-00059.safetensors",
+    "model.layers.5.block_sparse_moe.experts.2.w2.weight": "model-00006-of-00059.safetensors",
+    "model.layers.5.block_sparse_moe.experts.2.w3.weight": "model-00006-of-00059.safetensors",
+    "model.layers.5.block_sparse_moe.experts.3.w1.weight": "model-00006-of-00059.safetensors",
+    "model.layers.5.block_sparse_moe.experts.3.w2.weight": "model-00006-of-00059.safetensors",
+    "model.layers.5.block_sparse_moe.experts.3.w3.weight": "model-00006-of-00059.safetensors",
+    "model.layers.5.block_sparse_moe.experts.4.w1.weight": "model-00006-of-00059.safetensors",
+    "model.layers.5.block_sparse_moe.experts.4.w2.weight": "model-00006-of-00059.safetensors",
+    "model.layers.5.block_sparse_moe.experts.4.w3.weight": "model-00006-of-00059.safetensors",
+    "model.layers.5.block_sparse_moe.experts.5.w1.weight": "model-00006-of-00059.safetensors",
+    "model.layers.5.block_sparse_moe.experts.5.w2.weight": "model-00006-of-00059.safetensors",
+    "model.layers.5.block_sparse_moe.experts.5.w3.weight": "model-00007-of-00059.safetensors",
+    "model.layers.5.block_sparse_moe.experts.6.w1.weight": "model-00007-of-00059.safetensors",
+    "model.layers.5.block_sparse_moe.experts.6.w2.weight": "model-00007-of-00059.safetensors",
+    "model.layers.5.block_sparse_moe.experts.6.w3.weight": "model-00007-of-00059.safetensors",
+    "model.layers.5.block_sparse_moe.experts.7.w1.weight": "model-00007-of-00059.safetensors",
+    "model.layers.5.block_sparse_moe.experts.7.w2.weight": "model-00007-of-00059.safetensors",
+    "model.layers.5.block_sparse_moe.experts.7.w3.weight": "model-00007-of-00059.safetensors",
+    "model.layers.5.block_sparse_moe.gate.weight": "model-00006-of-00059.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00007-of-00059.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00007-of-00059.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00006-of-00059.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00006-of-00059.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00006-of-00059.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00006-of-00059.safetensors",
+    "model.layers.50.block_sparse_moe.experts.0.w1.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.block_sparse_moe.experts.0.w2.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.block_sparse_moe.experts.0.w3.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.block_sparse_moe.experts.1.w1.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.block_sparse_moe.experts.1.w2.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.block_sparse_moe.experts.1.w3.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.block_sparse_moe.experts.2.w1.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.block_sparse_moe.experts.2.w2.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.block_sparse_moe.experts.2.w3.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.block_sparse_moe.experts.3.w1.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.block_sparse_moe.experts.3.w2.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.block_sparse_moe.experts.3.w3.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.block_sparse_moe.experts.4.w1.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.block_sparse_moe.experts.4.w2.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.block_sparse_moe.experts.4.w3.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.block_sparse_moe.experts.5.w1.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.block_sparse_moe.experts.5.w2.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.block_sparse_moe.experts.5.w3.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.block_sparse_moe.experts.6.w1.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.block_sparse_moe.experts.6.w2.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.block_sparse_moe.experts.6.w3.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.block_sparse_moe.experts.7.w1.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.block_sparse_moe.experts.7.w2.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.block_sparse_moe.experts.7.w3.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.block_sparse_moe.gate.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.input_layernorm.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.post_attention_layernorm.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.self_attn.k_proj.weight": "model-00052-of-00059.safetensors",
+    "model.layers.50.self_attn.o_proj.weight": "model-00053-of-00059.safetensors",
+    "model.layers.50.self_attn.q_proj.weight": "model-00052-of-00059.safetensors",
+    "model.layers.50.self_attn.v_proj.weight": "model-00052-of-00059.safetensors",
+    "model.layers.51.block_sparse_moe.experts.0.w1.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.block_sparse_moe.experts.0.w2.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.block_sparse_moe.experts.0.w3.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.block_sparse_moe.experts.1.w1.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.block_sparse_moe.experts.1.w2.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.block_sparse_moe.experts.1.w3.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.block_sparse_moe.experts.2.w1.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.block_sparse_moe.experts.2.w2.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.block_sparse_moe.experts.2.w3.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.block_sparse_moe.experts.3.w1.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.block_sparse_moe.experts.3.w2.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.block_sparse_moe.experts.3.w3.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.block_sparse_moe.experts.4.w1.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.block_sparse_moe.experts.4.w2.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.block_sparse_moe.experts.4.w3.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.block_sparse_moe.experts.5.w1.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.block_sparse_moe.experts.5.w2.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.block_sparse_moe.experts.5.w3.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.block_sparse_moe.experts.6.w1.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.block_sparse_moe.experts.6.w2.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.block_sparse_moe.experts.6.w3.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.block_sparse_moe.experts.7.w1.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.block_sparse_moe.experts.7.w2.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.block_sparse_moe.experts.7.w3.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.block_sparse_moe.gate.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.input_layernorm.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.post_attention_layernorm.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.self_attn.k_proj.weight": "model-00053-of-00059.safetensors",
+    "model.layers.51.self_attn.o_proj.weight": "model-00054-of-00059.safetensors",
+    "model.layers.51.self_attn.q_proj.weight": "model-00053-of-00059.safetensors",
+    "model.layers.51.self_attn.v_proj.weight": "model-00054-of-00059.safetensors",
+    "model.layers.52.block_sparse_moe.experts.0.w1.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.block_sparse_moe.experts.0.w2.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.block_sparse_moe.experts.0.w3.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.block_sparse_moe.experts.1.w1.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.block_sparse_moe.experts.1.w2.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.block_sparse_moe.experts.1.w3.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.block_sparse_moe.experts.2.w1.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.block_sparse_moe.experts.2.w2.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.block_sparse_moe.experts.2.w3.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.block_sparse_moe.experts.3.w1.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.block_sparse_moe.experts.3.w2.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.block_sparse_moe.experts.3.w3.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.block_sparse_moe.experts.4.w1.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.block_sparse_moe.experts.4.w2.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.block_sparse_moe.experts.4.w3.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.block_sparse_moe.experts.5.w1.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.block_sparse_moe.experts.5.w2.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.block_sparse_moe.experts.5.w3.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.block_sparse_moe.experts.6.w1.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.block_sparse_moe.experts.6.w2.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.block_sparse_moe.experts.6.w3.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.block_sparse_moe.experts.7.w1.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.block_sparse_moe.experts.7.w2.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.block_sparse_moe.experts.7.w3.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.block_sparse_moe.gate.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.input_layernorm.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.post_attention_layernorm.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.self_attn.k_proj.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.self_attn.o_proj.weight": "model-00055-of-00059.safetensors",
+    "model.layers.52.self_attn.q_proj.weight": "model-00054-of-00059.safetensors",
+    "model.layers.52.self_attn.v_proj.weight": "model-00055-of-00059.safetensors",
+    "model.layers.53.block_sparse_moe.experts.0.w1.weight": "model-00056-of-00059.safetensors",
+    "model.layers.53.block_sparse_moe.experts.0.w2.weight": "model-00056-of-00059.safetensors",
+    "model.layers.53.block_sparse_moe.experts.0.w3.weight": "model-00056-of-00059.safetensors",
+    "model.layers.53.block_sparse_moe.experts.1.w1.weight": "model-00056-of-00059.safetensors",
+    "model.layers.53.block_sparse_moe.experts.1.w2.weight": "model-00056-of-00059.safetensors",
+    "model.layers.53.block_sparse_moe.experts.1.w3.weight": "model-00056-of-00059.safetensors",
+    "model.layers.53.block_sparse_moe.experts.2.w1.weight": "model-00056-of-00059.safetensors",
+    "model.layers.53.block_sparse_moe.experts.2.w2.weight": "model-00056-of-00059.safetensors",
+    "model.layers.53.block_sparse_moe.experts.2.w3.weight": "model-00056-of-00059.safetensors",
+    "model.layers.53.block_sparse_moe.experts.3.w1.weight": "model-00056-of-00059.safetensors",
+    "model.layers.53.block_sparse_moe.experts.3.w2.weight": "model-00056-of-00059.safetensors",
+    "model.layers.53.block_sparse_moe.experts.3.w3.weight": "model-00056-of-00059.safetensors",
+    "model.layers.53.block_sparse_moe.experts.4.w1.weight": "model-00056-of-00059.safetensors",
+    "model.layers.53.block_sparse_moe.experts.4.w2.weight": "model-00056-of-00059.safetensors",
+    "model.layers.53.block_sparse_moe.experts.4.w3.weight": "model-00056-of-00059.safetensors",
+    "model.layers.53.block_sparse_moe.experts.5.w1.weight": "model-00056-of-00059.safetensors",
+    "model.layers.53.block_sparse_moe.experts.5.w2.weight": "model-00056-of-00059.safetensors",
+    "model.layers.53.block_sparse_moe.experts.5.w3.weight": "model-00056-of-00059.safetensors",
+    "model.layers.53.block_sparse_moe.experts.6.w1.weight": "model-00056-of-00059.safetensors",
+    "model.layers.53.block_sparse_moe.experts.6.w2.weight": "model-00056-of-00059.safetensors",
+    "model.layers.53.block_sparse_moe.experts.6.w3.weight": "model-00056-of-00059.safetensors",
+    "model.layers.53.block_sparse_moe.experts.7.w1.weight": "model-00056-of-00059.safetensors",
+    "model.layers.53.block_sparse_moe.experts.7.w2.weight": "model-00056-of-00059.safetensors",
+    "model.layers.53.block_sparse_moe.experts.7.w3.weight": "model-00057-of-00059.safetensors",
+    "model.layers.53.block_sparse_moe.gate.weight": "model-00056-of-00059.safetensors",
+    "model.layers.53.input_layernorm.weight": "model-00057-of-00059.safetensors",
+    "model.layers.53.post_attention_layernorm.weight": "model-00057-of-00059.safetensors",
+    "model.layers.53.self_attn.k_proj.weight": "model-00056-of-00059.safetensors",
+    "model.layers.53.self_attn.o_proj.weight": "model-00056-of-00059.safetensors",
+    "model.layers.53.self_attn.q_proj.weight": "model-00056-of-00059.safetensors",
+    "model.layers.53.self_attn.v_proj.weight": "model-00056-of-00059.safetensors",
+    "model.layers.54.block_sparse_moe.experts.0.w1.weight": "model-00057-of-00059.safetensors",
+    "model.layers.54.block_sparse_moe.experts.0.w2.weight": "model-00057-of-00059.safetensors",
+    "model.layers.54.block_sparse_moe.experts.0.w3.weight": "model-00057-of-00059.safetensors",
+    "model.layers.54.block_sparse_moe.experts.1.w1.weight": "model-00057-of-00059.safetensors",
+    "model.layers.54.block_sparse_moe.experts.1.w2.weight": "model-00057-of-00059.safetensors",
+    "model.layers.54.block_sparse_moe.experts.1.w3.weight": "model-00057-of-00059.safetensors",
+    "model.layers.54.block_sparse_moe.experts.2.w1.weight": "model-00057-of-00059.safetensors",
+    "model.layers.54.block_sparse_moe.experts.2.w2.weight": "model-00057-of-00059.safetensors",
+    "model.layers.54.block_sparse_moe.experts.2.w3.weight": "model-00057-of-00059.safetensors",
+    "model.layers.54.block_sparse_moe.experts.3.w1.weight": "model-00057-of-00059.safetensors",
+    "model.layers.54.block_sparse_moe.experts.3.w2.weight": "model-00057-of-00059.safetensors",
+    "model.layers.54.block_sparse_moe.experts.3.w3.weight": "model-00057-of-00059.safetensors",
+    "model.layers.54.block_sparse_moe.experts.4.w1.weight": "model-00057-of-00059.safetensors",
+    "model.layers.54.block_sparse_moe.experts.4.w2.weight": "model-00057-of-00059.safetensors",
+    "model.layers.54.block_sparse_moe.experts.4.w3.weight": "model-00057-of-00059.safetensors",
+    "model.layers.54.block_sparse_moe.experts.5.w1.weight": "model-00057-of-00059.safetensors",
+    "model.layers.54.block_sparse_moe.experts.5.w2.weight": "model-00057-of-00059.safetensors",
+    "model.layers.54.block_sparse_moe.experts.5.w3.weight": "model-00057-of-00059.safetensors",
+    "model.layers.54.block_sparse_moe.experts.6.w1.weight": "model-00057-of-00059.safetensors",
+    "model.layers.54.block_sparse_moe.experts.6.w2.weight": "model-00057-of-00059.safetensors",
+    "model.layers.54.block_sparse_moe.experts.6.w3.weight": "model-00057-of-00059.safetensors",
+    "model.layers.54.block_sparse_moe.experts.7.w1.weight": "model-00057-of-00059.safetensors",
+    "model.layers.54.block_sparse_moe.experts.7.w2.weight": "model-00058-of-00059.safetensors",
+    "model.layers.54.block_sparse_moe.experts.7.w3.weight": "model-00058-of-00059.safetensors",
+    "model.layers.54.block_sparse_moe.gate.weight": "model-00057-of-00059.safetensors",
+    "model.layers.54.input_layernorm.weight": "model-00058-of-00059.safetensors",
+    "model.layers.54.post_attention_layernorm.weight": "model-00058-of-00059.safetensors",
+    "model.layers.54.self_attn.k_proj.weight": "model-00057-of-00059.safetensors",
+    "model.layers.54.self_attn.o_proj.weight": "model-00057-of-00059.safetensors",
+    "model.layers.54.self_attn.q_proj.weight": "model-00057-of-00059.safetensors",
+    "model.layers.54.self_attn.v_proj.weight": "model-00057-of-00059.safetensors",
+    "model.layers.55.block_sparse_moe.experts.0.w1.weight": "model-00058-of-00059.safetensors",
+    "model.layers.55.block_sparse_moe.experts.0.w2.weight": "model-00058-of-00059.safetensors",
+    "model.layers.55.block_sparse_moe.experts.0.w3.weight": "model-00058-of-00059.safetensors",
+    "model.layers.55.block_sparse_moe.experts.1.w1.weight": "model-00058-of-00059.safetensors",
+    "model.layers.55.block_sparse_moe.experts.1.w2.weight": "model-00058-of-00059.safetensors",
+    "model.layers.55.block_sparse_moe.experts.1.w3.weight": "model-00058-of-00059.safetensors",
+    "model.layers.55.block_sparse_moe.experts.2.w1.weight": "model-00058-of-00059.safetensors",
+    "model.layers.55.block_sparse_moe.experts.2.w2.weight": "model-00058-of-00059.safetensors",
+    "model.layers.55.block_sparse_moe.experts.2.w3.weight": "model-00058-of-00059.safetensors",
+    "model.layers.55.block_sparse_moe.experts.3.w1.weight": "model-00058-of-00059.safetensors",
+    "model.layers.55.block_sparse_moe.experts.3.w2.weight": "model-00058-of-00059.safetensors",
+    "model.layers.55.block_sparse_moe.experts.3.w3.weight": "model-00058-of-00059.safetensors",
+    "model.layers.55.block_sparse_moe.experts.4.w1.weight": "model-00058-of-00059.safetensors",
+    "model.layers.55.block_sparse_moe.experts.4.w2.weight": "model-00058-of-00059.safetensors",
+    "model.layers.55.block_sparse_moe.experts.4.w3.weight": "model-00058-of-00059.safetensors",
+    "model.layers.55.block_sparse_moe.experts.5.w1.weight": "model-00058-of-00059.safetensors",
+    "model.layers.55.block_sparse_moe.experts.5.w2.weight": "model-00058-of-00059.safetensors",
+    "model.layers.55.block_sparse_moe.experts.5.w3.weight": "model-00058-of-00059.safetensors",
+    "model.layers.55.block_sparse_moe.experts.6.w1.weight": "model-00058-of-00059.safetensors",
+    "model.layers.55.block_sparse_moe.experts.6.w2.weight": "model-00058-of-00059.safetensors",
+    "model.layers.55.block_sparse_moe.experts.6.w3.weight": "model-00058-of-00059.safetensors",
+    "model.layers.55.block_sparse_moe.experts.7.w1.weight": "model-00059-of-00059.safetensors",
+    "model.layers.55.block_sparse_moe.experts.7.w2.weight": "model-00059-of-00059.safetensors",
+    "model.layers.55.block_sparse_moe.experts.7.w3.weight": "model-00059-of-00059.safetensors",
+    "model.layers.55.block_sparse_moe.gate.weight": "model-00058-of-00059.safetensors",
+    "model.layers.55.input_layernorm.weight": "model-00059-of-00059.safetensors",
+    "model.layers.55.post_attention_layernorm.weight": "model-00059-of-00059.safetensors",
+    "model.layers.55.self_attn.k_proj.weight": "model-00058-of-00059.safetensors",
+    "model.layers.55.self_attn.o_proj.weight": "model-00058-of-00059.safetensors",
+    "model.layers.55.self_attn.q_proj.weight": "model-00058-of-00059.safetensors",
+    "model.layers.55.self_attn.v_proj.weight": "model-00058-of-00059.safetensors",
+    "model.layers.6.block_sparse_moe.experts.0.w1.weight": "model-00007-of-00059.safetensors",
+    "model.layers.6.block_sparse_moe.experts.0.w2.weight": "model-00007-of-00059.safetensors",
+    "model.layers.6.block_sparse_moe.experts.0.w3.weight": "model-00007-of-00059.safetensors",
+    "model.layers.6.block_sparse_moe.experts.1.w1.weight": "model-00007-of-00059.safetensors",
+    "model.layers.6.block_sparse_moe.experts.1.w2.weight": "model-00007-of-00059.safetensors",
+    "model.layers.6.block_sparse_moe.experts.1.w3.weight": "model-00007-of-00059.safetensors",
+    "model.layers.6.block_sparse_moe.experts.2.w1.weight": "model-00007-of-00059.safetensors",
+    "model.layers.6.block_sparse_moe.experts.2.w2.weight": "model-00007-of-00059.safetensors",
+    "model.layers.6.block_sparse_moe.experts.2.w3.weight": "model-00007-of-00059.safetensors",
+    "model.layers.6.block_sparse_moe.experts.3.w1.weight": "model-00007-of-00059.safetensors",
+    "model.layers.6.block_sparse_moe.experts.3.w2.weight": "model-00007-of-00059.safetensors",
+    "model.layers.6.block_sparse_moe.experts.3.w3.weight": "model-00007-of-00059.safetensors",
+    "model.layers.6.block_sparse_moe.experts.4.w1.weight": "model-00007-of-00059.safetensors",
+    "model.layers.6.block_sparse_moe.experts.4.w2.weight": "model-00007-of-00059.safetensors",
+    "model.layers.6.block_sparse_moe.experts.4.w3.weight": "model-00007-of-00059.safetensors",
+    "model.layers.6.block_sparse_moe.experts.5.w1.weight": "model-00007-of-00059.safetensors",
+    "model.layers.6.block_sparse_moe.experts.5.w2.weight": "model-00008-of-00059.safetensors",
+    "model.layers.6.block_sparse_moe.experts.5.w3.weight": "model-00008-of-00059.safetensors",
+    "model.layers.6.block_sparse_moe.experts.6.w1.weight": "model-00008-of-00059.safetensors",
+    "model.layers.6.block_sparse_moe.experts.6.w2.weight": "model-00008-of-00059.safetensors",
+    "model.layers.6.block_sparse_moe.experts.6.w3.weight": "model-00008-of-00059.safetensors",
+    "model.layers.6.block_sparse_moe.experts.7.w1.weight": "model-00008-of-00059.safetensors",
+    "model.layers.6.block_sparse_moe.experts.7.w2.weight": "model-00008-of-00059.safetensors",
+    "model.layers.6.block_sparse_moe.experts.7.w3.weight": "model-00008-of-00059.safetensors",
+    "model.layers.6.block_sparse_moe.gate.weight": "model-00007-of-00059.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00008-of-00059.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00008-of-00059.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00007-of-00059.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00007-of-00059.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00007-of-00059.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00007-of-00059.safetensors",
+    "model.layers.7.block_sparse_moe.experts.0.w1.weight": "model-00008-of-00059.safetensors",
+    "model.layers.7.block_sparse_moe.experts.0.w2.weight": "model-00008-of-00059.safetensors",
+    "model.layers.7.block_sparse_moe.experts.0.w3.weight": "model-00008-of-00059.safetensors",
+    "model.layers.7.block_sparse_moe.experts.1.w1.weight": "model-00008-of-00059.safetensors",
+    "model.layers.7.block_sparse_moe.experts.1.w2.weight": "model-00008-of-00059.safetensors",
+    "model.layers.7.block_sparse_moe.experts.1.w3.weight": "model-00008-of-00059.safetensors",
+    "model.layers.7.block_sparse_moe.experts.2.w1.weight": "model-00008-of-00059.safetensors",
+    "model.layers.7.block_sparse_moe.experts.2.w2.weight": "model-00008-of-00059.safetensors",
+    "model.layers.7.block_sparse_moe.experts.2.w3.weight": "model-00008-of-00059.safetensors",
+    "model.layers.7.block_sparse_moe.experts.3.w1.weight": "model-00008-of-00059.safetensors",
+    "model.layers.7.block_sparse_moe.experts.3.w2.weight": "model-00008-of-00059.safetensors",
+    "model.layers.7.block_sparse_moe.experts.3.w3.weight": "model-00008-of-00059.safetensors",
+    "model.layers.7.block_sparse_moe.experts.4.w1.weight": "model-00008-of-00059.safetensors",
+    "model.layers.7.block_sparse_moe.experts.4.w2.weight": "model-00008-of-00059.safetensors",
+    "model.layers.7.block_sparse_moe.experts.4.w3.weight": "model-00008-of-00059.safetensors",
+    "model.layers.7.block_sparse_moe.experts.5.w1.weight": "model-00009-of-00059.safetensors",
+    "model.layers.7.block_sparse_moe.experts.5.w2.weight": "model-00009-of-00059.safetensors",
+    "model.layers.7.block_sparse_moe.experts.5.w3.weight": "model-00009-of-00059.safetensors",
+    "model.layers.7.block_sparse_moe.experts.6.w1.weight": "model-00009-of-00059.safetensors",
+    "model.layers.7.block_sparse_moe.experts.6.w2.weight": "model-00009-of-00059.safetensors",
+    "model.layers.7.block_sparse_moe.experts.6.w3.weight": "model-00009-of-00059.safetensors",
+    "model.layers.7.block_sparse_moe.experts.7.w1.weight": "model-00009-of-00059.safetensors",
+    "model.layers.7.block_sparse_moe.experts.7.w2.weight": "model-00009-of-00059.safetensors",
+    "model.layers.7.block_sparse_moe.experts.7.w3.weight": "model-00009-of-00059.safetensors",
+    "model.layers.7.block_sparse_moe.gate.weight": "model-00008-of-00059.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00009-of-00059.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00009-of-00059.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00008-of-00059.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00008-of-00059.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00008-of-00059.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00008-of-00059.safetensors",
+    "model.layers.8.block_sparse_moe.experts.0.w1.weight": "model-00009-of-00059.safetensors",
+    "model.layers.8.block_sparse_moe.experts.0.w2.weight": "model-00009-of-00059.safetensors",
+    "model.layers.8.block_sparse_moe.experts.0.w3.weight": "model-00009-of-00059.safetensors",
+    "model.layers.8.block_sparse_moe.experts.1.w1.weight": "model-00009-of-00059.safetensors",
+    "model.layers.8.block_sparse_moe.experts.1.w2.weight": "model-00009-of-00059.safetensors",
+    "model.layers.8.block_sparse_moe.experts.1.w3.weight": "model-00009-of-00059.safetensors",
+    "model.layers.8.block_sparse_moe.experts.2.w1.weight": "model-00009-of-00059.safetensors",
+    "model.layers.8.block_sparse_moe.experts.2.w2.weight": "model-00009-of-00059.safetensors",
+    "model.layers.8.block_sparse_moe.experts.2.w3.weight": "model-00009-of-00059.safetensors",
+    "model.layers.8.block_sparse_moe.experts.3.w1.weight": "model-00009-of-00059.safetensors",
+    "model.layers.8.block_sparse_moe.experts.3.w2.weight": "model-00009-of-00059.safetensors",
+    "model.layers.8.block_sparse_moe.experts.3.w3.weight": "model-00009-of-00059.safetensors",
+    "model.layers.8.block_sparse_moe.experts.4.w1.weight": "model-00009-of-00059.safetensors",
+    "model.layers.8.block_sparse_moe.experts.4.w2.weight": "model-00009-of-00059.safetensors",
+    "model.layers.8.block_sparse_moe.experts.4.w3.weight": "model-00010-of-00059.safetensors",
+    "model.layers.8.block_sparse_moe.experts.5.w1.weight": "model-00010-of-00059.safetensors",
+    "model.layers.8.block_sparse_moe.experts.5.w2.weight": "model-00010-of-00059.safetensors",
+    "model.layers.8.block_sparse_moe.experts.5.w3.weight": "model-00010-of-00059.safetensors",
+    "model.layers.8.block_sparse_moe.experts.6.w1.weight": "model-00010-of-00059.safetensors",
+    "model.layers.8.block_sparse_moe.experts.6.w2.weight": "model-00010-of-00059.safetensors",
+    "model.layers.8.block_sparse_moe.experts.6.w3.weight": "model-00010-of-00059.safetensors",
+    "model.layers.8.block_sparse_moe.experts.7.w1.weight": "model-00010-of-00059.safetensors",
+    "model.layers.8.block_sparse_moe.experts.7.w2.weight": "model-00010-of-00059.safetensors",
+    "model.layers.8.block_sparse_moe.experts.7.w3.weight": "model-00010-of-00059.safetensors",
+    "model.layers.8.block_sparse_moe.gate.weight": "model-00009-of-00059.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00010-of-00059.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00010-of-00059.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00009-of-00059.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00009-of-00059.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00009-of-00059.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00009-of-00059.safetensors",
+    "model.layers.9.block_sparse_moe.experts.0.w1.weight": "model-00010-of-00059.safetensors",
+    "model.layers.9.block_sparse_moe.experts.0.w2.weight": "model-00010-of-00059.safetensors",
+    "model.layers.9.block_sparse_moe.experts.0.w3.weight": "model-00010-of-00059.safetensors",
+    "model.layers.9.block_sparse_moe.experts.1.w1.weight": "model-00010-of-00059.safetensors",
+    "model.layers.9.block_sparse_moe.experts.1.w2.weight": "model-00010-of-00059.safetensors",
+    "model.layers.9.block_sparse_moe.experts.1.w3.weight": "model-00010-of-00059.safetensors",
+    "model.layers.9.block_sparse_moe.experts.2.w1.weight": "model-00010-of-00059.safetensors",
+    "model.layers.9.block_sparse_moe.experts.2.w2.weight": "model-00010-of-00059.safetensors",
+    "model.layers.9.block_sparse_moe.experts.2.w3.weight": "model-00010-of-00059.safetensors",
+    "model.layers.9.block_sparse_moe.experts.3.w1.weight": "model-00010-of-00059.safetensors",
+    "model.layers.9.block_sparse_moe.experts.3.w2.weight": "model-00010-of-00059.safetensors",
+    "model.layers.9.block_sparse_moe.experts.3.w3.weight": "model-00010-of-00059.safetensors",
+    "model.layers.9.block_sparse_moe.experts.4.w1.weight": "model-00010-of-00059.safetensors",
+    "model.layers.9.block_sparse_moe.experts.4.w2.weight": "model-00011-of-00059.safetensors",
+    "model.layers.9.block_sparse_moe.experts.4.w3.weight": "model-00011-of-00059.safetensors",
+    "model.layers.9.block_sparse_moe.experts.5.w1.weight": "model-00011-of-00059.safetensors",
+    "model.layers.9.block_sparse_moe.experts.5.w2.weight": "model-00011-of-00059.safetensors",
+    "model.layers.9.block_sparse_moe.experts.5.w3.weight": "model-00011-of-00059.safetensors",
+    "model.layers.9.block_sparse_moe.experts.6.w1.weight": "model-00011-of-00059.safetensors",
+    "model.layers.9.block_sparse_moe.experts.6.w2.weight": "model-00011-of-00059.safetensors",
+    "model.layers.9.block_sparse_moe.experts.6.w3.weight": "model-00011-of-00059.safetensors",
+    "model.layers.9.block_sparse_moe.experts.7.w1.weight": "model-00011-of-00059.safetensors",
+    "model.layers.9.block_sparse_moe.experts.7.w2.weight": "model-00011-of-00059.safetensors",
+    "model.layers.9.block_sparse_moe.experts.7.w3.weight": "model-00011-of-00059.safetensors",
+    "model.layers.9.block_sparse_moe.gate.weight": "model-00010-of-00059.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00011-of-00059.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00011-of-00059.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00010-of-00059.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00010-of-00059.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00010-of-00059.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00010-of-00059.safetensors",
+    "model.norm.weight": "model-00059-of-00059.safetensors"
+  }
+}