Upload folder using huggingface_hub

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,8 @@
+---
+license: apache-2.0
+library_name: transformers
+---
+
+Unofficial dequantized weight of [grok-1](https://huggingface.co/xai-org/grok-1) in HF Transformers format.
+
+The weights are converted using the [script here](https://gist.github.com/chu-tianxiang/ec310e15d56949fd0f351cb5f65ee7a1) ran inside the [grok-1 repo](https://github.com/xai-org/grok-1). Since downloading the dequantized weight needs twice as much time, it's recommended to download the original weight and convert on your own.

__init__.py

@@ -0,0 +1,2 @@
+from .configuration_grok import *
+from .modeling_grok import *

config.json

@@ -0,0 +1,37 @@
+{
+  "_name_or_path": "grok-ztmp",
+  "architectures": [
+    "GrokForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "attn_output_multiplier": 0.08838834764831845,
+  "auto_map": {
+    "AutoConfig": "configuration_grok.GrokConfig",
+    "AutoModelForCausalLM": "modeling_grok.GrokForCausalLM"
+  },
+  "bos_token_id": 1,
+  "embedding_multiplier_scale": 78.38367176906169,
+  "eos_token_id": 2,
+  "hidden_act": "gelu_new",
+  "hidden_size": 6144,
+  "initializer_range": 0.02,
+  "intermediate_size": 32768,
+  "max_position_embeddings": 8192,
+  "model_type": "grok",
+  "num_attention_heads": 48,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 64,
+  "num_key_value_heads": 8,
+  "num_local_experts": 8,
+  "output_multiplier_scale": 0.5773502691896257,
+  "output_router_logits": false,
+  "pad_token_id": 0,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "router_aux_loss_coef": 0.02,
+  "sliding_window": null,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.38.2",
+  "use_cache": true,
+  "vocab_size": 131072
+}

configuration_grok.py

@@ -0,0 +1,151 @@
+# coding=utf-8
+# Copyright 2023 Mixtral AI and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Grok model configuration"""
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+
+class GrokConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`GrokModel`]. 
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+
+    Args:
+        vocab_size (`int`, *optional*, defaults to 32000):
+            Vocabulary size of the Mixtral model. Defines the number of different tokens that can be represented by the
+            `inputs_ids` passed when calling [`MixtralModel`]
+        hidden_size (`int`, *optional*, defaults to 4096):
+            Dimension of the hidden representations.
+        intermediate_size (`int`, *optional*, defaults to 14336):
+            Dimension of the MLP representations.
+        num_hidden_layers (`int`, *optional*, defaults to 32):
+            Number of hidden layers in the Transformer encoder.
+        num_attention_heads (`int`, *optional*, defaults to 32):
+            Number of attention heads for each attention layer in the Transformer encoder.
+        num_key_value_heads (`int`, *optional*, defaults to 8):
+            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+            `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+            by meanpooling all the original heads within that group. For more details checkout [this
+            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `8`.
+        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+            The non-linear activation function (function or string) in the decoder.
+        max_position_embeddings (`int`, *optional*, defaults to `4096*32`):
+            The maximum sequence length that this model might ever be used with. Mixtral's sliding window attention
+            allows sequence of up to 4096*32 tokens.
+        initializer_range (`float`, *optional*, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        rms_norm_eps (`float`, *optional*, defaults to 1e-05):
+            The epsilon used by the rms normalization layers.
+        use_cache (`bool`, *optional*, defaults to `True`):
+            Whether or not the model should return the last key/values attentions (not used by all models). Only
+            relevant if `config.is_decoder=True`.
+        pad_token_id (`int`, *optional*):
+            The id of the padding token.
+        bos_token_id (`int`, *optional*, defaults to 1):
+            The id of the "beginning-of-sequence" token.
+        eos_token_id (`int`, *optional*, defaults to 2):
+            The id of the "end-of-sequence" token.
+        tie_word_embeddings (`bool`, *optional*, defaults to `True`):
+            Whether the model's input and output word embeddings should be tied.
+        rope_theta (`float`, *optional*, defaults to 100000.0):
+            The base period of the RoPE embeddings.
+        attention_dropout (`float`, *optional*, defaults to 0.0):
+            The dropout ratio for the attention probabilities.
+        num_experts_per_tok (`int`, *optional*, defaults to 2):
+            The number of experts to root per-token, can be also interpreted as the `top-p` routing
+            parameter
+        num_local_experts (`int`, *optional*, defaults to 8):
+            Number of experts per Sparse MLP layer.
+        output_router_logits (`bool`, *optional*, defaults to `False`):
+            Whether or not the router logits should be returned by the model. Enabeling this will also
+            allow the model to output the auxiliary loss. See [here]() for more details
+        router_aux_loss_coef (`float`, *optional*, defaults to 0.001):
+            The aux loss factor for the total loss.
+
+    """
+
+    model_type = "grok"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        vocab_size=131072,
+        hidden_size=6144,
+        intermediate_size=32768,
+        num_hidden_layers=64,
+        num_attention_heads=48,
+        num_key_value_heads=8,
+        hidden_act="silu",
+        max_position_embeddings=4096,
+        initializer_range=0.02,
+        rms_norm_eps=1e-5,
+        use_cache=True,
+        pad_token_id=0,
+        bos_token_id=1,
+        eos_token_id=2,
+        tie_word_embeddings=True,
+        rope_theta=1e5,
+        attention_dropout=0.0,
+        num_experts_per_tok=2,
+        num_local_experts=8,
+        output_router_logits=False,
+        router_aux_loss_coef=0.001,
+        output_multiplier_scale=0.5773502691896257,
+        embedding_multiplier_scale=78.38367176906169,
+        attn_output_multiplier=0.08838834764831845,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+
+        # for backward compatibility
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.attention_dropout = attention_dropout
+
+        self.num_experts_per_tok = num_experts_per_tok
+        self.num_local_experts = num_local_experts
+        self.output_router_logits = output_router_logits
+        self.router_aux_loss_coef = router_aux_loss_coef
+        self.output_multiplier_scale = output_multiplier_scale
+       	self.embedding_multiplier_scale = embedding_multiplier_scale
+       	self.attn_output_multiplier = attn_output_multiplier
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 0,
+  "transformers_version": "4.38.2"
+}

model-00001-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:739afdc4fe7c736298f1056131da2a27d5e08e2d9d3654470866d11339e71d08
+size 4605544112

model-00002-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2da2697d8179839e2c4eab0df9f9af6bfe1ac0f022db6ee942e8e472d3b72e2b
+size 4831839776

model-00003-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c619d81a1e82abfaf498cd6cb4fefd509efccaca6e927fc62cc5060d9f6d6361
+size 4605643240

model-00004-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f77124ab7ec6a9fce4a42eef6908136fd6ef684a78bb157819d6215bdee9790
+size 4831839776

model-00005-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34babd1fbb246f160ee65e8d66963fa6dabe7d67bd6442f6a3b7e1120908d5e8
+size 4605643248

model-00006-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9741b2416238bac11d893af8c13e5f7bd824cfbbda7e53806e2e7432a42bd7c
+size 4831839776

model-00007-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b119ac6352864065f756bdd989b87a365dcedbfbd2285b306b2a81ef0a194901
+size 4605643240

model-00008-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:099d554f67f605c85f3a9e90cc6d781b252c46b2a4cc6c13fa3573ac13dd5671
+size 4831839776

model-00009-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11a8593548b70f975b3b08faec2c0a51ee2426a40e3590dcc20dd3b103075b39
+size 4605643240

model-00010-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e450cb6a0a6d23b2cc29cba9b0bdcdff2bd088fad8a5dc7064276640f87de16
+size 4831839776

model-00011-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f43f5d81e191360353b909503bb5257872884dc810c09331ad3ffcdd3e896356
+size 4605643248

model-00012-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4e3e6ecd078f12184f9b9d2369215cbaf6d9dbcd47f556f9e0c720efb1c848b
+size 4831839776

model-00013-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eac368881aedb951ee3eae29809f3c2bd8440f9ce7e76d74f2ff43c06377a1e7
+size 4605643240

model-00014-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16ff28e12db93b60248c80d47703b75a8ae8cc99fdf6dc28e2972ccda76aab8c
+size 4831839776

model-00015-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:772707d8d2b363e9fe2ece49d26b099f57d7d3be0d0c63b85d7d399317391af6
+size 4605643240

model-00016-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8d31ef102baf09a2b05f466eb411333070070c8cc6e0e878c503ad53bb3a26d
+size 4831839776

model-00017-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14b32ffd8b1fb73af472141f2144b8c2832a0d05bd8c3c7719268082d3126648
+size 4932602224

model-00018-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e16a2317e13ff084cf5db91213c073d1228aff1c05eb5e05a5073602eee1b75
+size 4907534104

model-00019-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4e05340708865242537b03a520a934f91dac42cadca279420d1510ffbbbf714
+size 4932602224

model-00020-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f0469775fc7cff1e86c415d04a130ee6e5d279ac40ff65971b8062328036cad
+size 4907534104

model-00021-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c3bcac033018cd66c6b61f4c1694f5bb3f8f82908482c1d531ccb4e7176a48c
+size 4932602216

model-00022-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f39177735f9e11cc0547eaf56af0b7d2977864d7caae927bbb29aca14aa63ed
+size 4907534120

model-00023-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13661a056515aaada2be4620a02784080d7fbbbc79c190b19b6aebe7613951b1
+size 4932602248

model-00024-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:755229210823b40a3241880975d6fc51771b7cc3c36e54219fb44dbed48cb2c2
+size 4907534120

model-00025-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8119c0587696df0b7a9c74758df30587657bd1cd766d105020c590c9bd035c0
+size 4932602248

model-00026-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c99121391ec30bb015010fbefc321f5fb7ca3bb58327261fb51e92edde042198
+size 4907534120

model-00027-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6d3e3effb57f725f3d078e2fffdca481bcd8bd9f0719bd88fc6e00ff60db9de
+size 4932602248

model-00028-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6924fd54f346d241b7e747d6118fac999acb8ed4253b5d156555ce28df9c9e2
+size 4907534120

model-00029-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54c8fcf7dfe74ffe480cc2673485a1ffe3c21dea79d109a27294118d511221db
+size 4932602248

model-00030-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a5da644297e20c4d3789b83ad856b0529940352e29f5b53999aaa5ad420b780
+size 4907534120

model-00031-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f0263c63393d929a3088eafaf071e30d53e4786c23e37af955e910693f0a482
+size 4932602248

model-00032-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f25b07be35d6b22d2cefddef443f020ecdb6f68a29fa0d0821c68b5ba471f126
+size 4907534120

model-00033-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b0ff57efd777bf516e8501f2fe41afac4f5b7cce723c246586238c33b9ee8f0
+size 4932602248

model-00034-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd742e0639c16b9e79b1360a23d8823e661e1e36ac21581dc4da9bca1f482e03
+size 4907534120

model-00035-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8646bc069c02ce86375f50572bcbcb65299a8f5a7c34be1164d983e9478e5b3
+size 4932602248

model-00036-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e376e4fb2c76cf03c335a7904872306935c76b3b69af8e6d4f6ae9a2fd87b6cf
+size 4907534120

model-00037-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55c188265cc7cc166ef111d1e92a4392cc2ab993d84046bae390bfdc5be28fb2
+size 4932602248

model-00038-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08309696ef312a305b8f25bb19f2781f086d1bd9b847a5232e1357482fb015bd
+size 4907534120

model-00039-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25bf348553f9229c981a48155bcccece55e9d7e7771de6ead7e512173c19a8bb
+size 4932602248

model-00040-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08c4f2b7b66e2b58060f3e42bca18ca397e61d806137f88448219e924f5b3a97
+size 4907534120

model-00041-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26c6502621af02d5354b715f0694021e26d9691a40b410d4a64899619993db4a
+size 4932602248

model-00042-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f61c9740cc478a478ce13122d71a91c426128d61b10effa0d7272b0b025a969a
+size 4907534120

model-00043-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6f9b9088299b99cd7fda366c803886bebcd58157d74456a0ae907c96199073c
+size 4932602248

model-00044-of-00129.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd1e334af06cab85e057b1be0ff963070991c36742a1ae3787c8465a9d22bce6
+size 4907534120