Upload ReplitLM

config.json

@@ -0,0 +1,46 @@
+{
+  "_name_or_path": "replit/replit-code-v1-3b",
+  "alibi": true,
+  "alibi_bias_max": 8,
+  "architectures": [
+    "ReplitLM"
+  ],
+  "attn_clip_qkv": null,
+  "attn_impl": "torch",
+  "attn_pdrop": 0,
+  "attn_qk_ln": false,
+  "attn_uses_sequence_id": false,
+  "auto_map": {
+    "AutoConfig": "configuration_replit_lm.ReplitLMConfig",
+    "AutoModelForCausalLM": "replit_lm.ReplitLM"
+  },
+  "d_model": 2560,
+  "emb_init_std": null,
+  "emb_init_uniform_lim": null,
+  "emb_pdrop": 0,
+  "embedding_fraction": 1.0,
+  "fan_mode": "fan_in",
+  "init_device": "cpu",
+  "init_div_is_residual": true,
+  "init_gain": 0,
+  "init_nonlinearity": "relu",
+  "init_std": 0.02,
+  "logit_scale": null,
+  "low_precision_layernorm": true,
+  "max_seq_len": 2048,
+  "mlp_ratio": 4,
+  "model_type": "replit_lm",
+  "n_heads": 32,
+  "n_layers": 32,
+  "no_bias": true,
+  "param_init_fn": "kaiming_normal_",
+  "prefix_lm": false,
+  "resid_pdrop": 0,
+  "softmax_scale": null,
+  "tokenizer_name": "replit/replit-code-v1-3b",
+  "torch_dtype": "float32",
+  "transformers_version": "4.28.1",
+  "use_cache": false,
+  "verbose": 0,
+  "vocab_size": 32768
+}

configuration_replit_lm.py

@@ -0,0 +1,168 @@
+# Copyright 2022 MosaicML Examples authors
+# SPDX-License-Identifier: Apache-2.0
+
+"""Forked for ReplitLM"""
+
+"""A HuggingFace-style model configuration."""
+
+
+from typing import Optional, Tuple, Union
+from transformers import PretrainedConfig
+class ReplitLMConfig(PretrainedConfig):
+    model_type = 'replit_lm'
+
+    def __init__(
+        self,
+        d_model: int = 2048,
+        n_heads: int = 16,
+        n_layers: int = 24,
+        mlp_ratio: int = 4,
+        max_seq_len: int = 2048,
+        vocab_size: int = 50368,
+        attn_pdrop: float = 0.0,
+        resid_pdrop: float = 0.0,
+        emb_pdrop: float = 0.0,
+        attn_impl: str = 'triton',
+        attn_qk_ln: bool = False,
+        attn_clip_qkv: Optional[float] = None,
+        softmax_scale: Optional[float] = None,
+        prefix_lm: Optional[bool] = False,
+        attn_uses_sequence_id: Optional[bool] = False,
+        alibi: bool = False,
+        alibi_bias_max: int = 8,
+        init_device: str = 'cpu',
+        logit_scale: Optional[Union[float, str]] = None,
+        no_bias: bool = False,
+        verbose: int = 0,
+        param_init_fn: str = 'kaiming_normal_',
+        init_div_is_residual: Union[int, float, str, bool] = True,
+        init_std: float = 0.02,
+        emb_init_std: Optional[float] = None,
+        emb_init_uniform_lim: Optional[Union[Tuple[float, float],
+                                             float]] = None,
+        init_gain: float = 0,
+        fan_mode: str = 'fan_in',
+        init_nonlinearity: str = 'relu',
+        embedding_fraction: float = 1.0,
+        low_precision_layernorm: bool = True,
+        use_cache: bool = False,
+        **kwargs,
+    ):
+        """The ReplitLM configuration class.
+
+        Args:
+            d_model (int): The size of the embedding dimension of the model.
+            n_heads (int): The number of attention heads.
+            n_layers (int): The number of layers in the model.
+            mlp_ratio (int): The ratio of the up/down scale in the MLP.
+            max_seq_len (int): The maximum sequence length of the model.
+            vocab_size (int): The size of the vocabulary.
+            attn_pdrop (float): The dropout probability for the attention layers.
+            resid_pdrop (float): The dropout probability applied to the attention output before combining with residual.
+            emb_pdrop (float): The dropout probability for the embedding layer.
+            attn_impl (str): The attention implementation to use. One of 'torch', 'flash', or 'triton'.
+            attn_qk_ln (bool): Whether to apply layer normalization to the queries and keys in the attention layer.
+            attn_clip_qkv (Optional[float]): If not None, clip the queries, keys, and values in the attention layer to
+                this value.
+            softmax_scale (Optional[float]): If not None, scale the softmax in the attention layer by this value. If None,
+                use the default scale of ``1/sqrt(d_keys)``.
+            prefix_lm (Optional[bool]): Whether the model should operate as a Prefix LM. This requires passing an
+                extra `prefix_mask` argument which indicates which tokens belong to the prefix. Tokens in the prefix
+                can attend to one another bi-directionally. Tokens outside the prefix use causal attention.
+            attn_uses_sequence_id (Optional[bool]): Whether to restrict attention to tokens that have the same sequence_id.
+                When the model is in `train` mode, this requires passing an extra `sequence_id` argument which indicates
+                which sub-sequence each token belongs to.
+                Defaults to ``False`` meaning any provided `sequence_id` will be ignored.
+            alibi (bool): Whether to use the alibi bias instead of position embeddings.
+            alibi_bias_max (int): The maximum value of the alibi bias.
+            init_device (str): The device to use for parameter initialization.
+            logit_scale (Optional[Union[float, str]]): If not None, scale the logits by this value.
+            no_bias (bool): Whether to use bias in all layers.
+            verbose (int): The verbosity level. 0 is silent.
+            param_init_fn (str): The parameter initialization scheme to use. One of 'default_', 'baseline_', 'kaiming_uniform_',
+                'kaiming_normal_', 'neox_init_', 'small_init_', 'xavier_uniform_', or 'xavier_normal_'.
+            init_div_is_residual (Union[int, float, str, bool]): Value to divide initial weights by if ``module._is_residual`` is True.
+            init_std (float): The standard deviation of the normal distribution used to initialize the model,
+                if using the baseline_ parameter initialization scheme.
+            emb_init_std (Optional[float]): The standard deviation of the normal distribution used to initialize the embedding layer.
+            emb_init_uniform_lim (Optional[Union[Tuple[float, float], float]]): The lower and upper limits of the uniform distribution
+                used to initialize the embedding layer. Mutually exclusive with ``emb_init_std``.
+            init_gain (float): The gain to use for parameter initialization with kaiming or xavier initialization schemes.
+            fan_mode (str): The fan mode to use for parameter initialization with kaiming initialization schemes.
+            init_nonlinearity (str): The nonlinearity to use for parameter initialization with kaiming initialization schemes.
+            embedding_fraction (float): The fraction to scale the gradients of the embedding layer by.
+            low_precision_layernorm (bool): Whether to use low precision layer normalization.
+            use_cache (bool): Whether or not the model should return the last key/values attentions
+        """
+        self.d_model = d_model
+        self.n_heads = n_heads
+        self.n_layers = n_layers
+        self.mlp_ratio = mlp_ratio
+        self.max_seq_len = max_seq_len
+        self.vocab_size = vocab_size
+        self.attn_pdrop = attn_pdrop
+        self.resid_pdrop = resid_pdrop
+        self.emb_pdrop = emb_pdrop
+        self.attn_impl = attn_impl
+        self.attn_qk_ln = attn_qk_ln
+        self.attn_clip_qkv = attn_clip_qkv
+        self.softmax_scale = softmax_scale
+        self.prefix_lm = prefix_lm
+        self.attn_uses_sequence_id = attn_uses_sequence_id
+        self.alibi = alibi
+        self.alibi_bias_max = alibi_bias_max
+        self.init_device = init_device
+        self.logit_scale = logit_scale
+        self.no_bias = no_bias
+        self.verbose = verbose
+        self.param_init_fn = param_init_fn
+        self.init_div_is_residual = init_div_is_residual
+        self.init_std = init_std
+        self.emb_init_std = emb_init_std
+        self.emb_init_uniform_lim = emb_init_uniform_lim
+        self.init_std = init_std
+        self.init_gain = init_gain
+        self.fan_mode = fan_mode
+        self.init_nonlinearity = init_nonlinearity
+        self.embedding_fraction = embedding_fraction
+        self.low_precision_layernorm = low_precision_layernorm
+        self.use_cache = use_cache
+        if 'name' in kwargs:
+            del kwargs['name']
+        if 'loss_fn' in kwargs:
+            del kwargs['loss_fn']
+        super().__init__(**kwargs)
+
+        self._validate_config()
+
+    def _validate_config(self):
+        if self.d_model % self.n_heads != 0:
+            raise ValueError('d_model must be divisible by n_heads')
+        if any(prob < 0 or prob > 1
+               for prob in [self.attn_pdrop, self.resid_pdrop, self.emb_pdrop]):
+            raise ValueError(
+                'attn_pdrop, resid_pdrop, emb_pdrop are probabilities and must be between 0 and 1'
+            )
+        if self.attn_impl not in ['torch', 'flash', 'triton']:
+            raise ValueError(f'Unknown attn_impl={self.attn_impl}')
+        if self.prefix_lm and self.attn_impl not in ['torch', 'triton']:
+            raise NotImplementedError(
+                'prefix_lm only implemented with torch and triton attention.')
+        if self.alibi and self.attn_impl not in ['torch', 'triton']:
+            raise NotImplementedError(
+                'alibi only implemented with torch and triton attention.')
+        if self.attn_uses_sequence_id and self.attn_impl not in [
+                'torch', 'triton'
+        ]:
+            raise NotImplementedError(
+                'attn_uses_sequence_id only implemented with torch and triton attention.'
+            )
+        if self.embedding_fraction > 1 or self.embedding_fraction <= 0:
+            raise ValueError(
+                'model.embedding_fraction must be between 0 (exclusive) and 1 (inclusive)!'
+            )
+        if isinstance(self.logit_scale,
+                      str) and self.logit_scale != 'inv_sqrt_d_model':
+            raise ValueError(
+                f"{self.logit_scale=} is not recognized as an option; use numeric value or 'inv_sqrt_d_model'."
+            )

generation_config.json

@@ -0,0 +1,5 @@
+{
+  "_from_model_config": true,
+  "transformers_version": "4.28.1",
+  "use_cache": false
+}

pytorch_model-00001-of-00002.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ace7a3b244a7802aec01e040880c6f1bd2bfb5099732ba2e30b164e9ba9c987d
+size 9983144733

pytorch_model-00002-of-00002.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:962355661d4729fa5c6529699fcea195da3b79cadd346f42dcb7678d82c96e48
+size 419464289

pytorch_model.bin.index.json

@@ -0,0 +1,201 @@
+{
+  "metadata": {
+    "total_size": 10402539520
+  },
+  "weight_map": {
+    "transformer.blocks.0.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.0.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.0.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.0.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.0.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.0.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.1.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.1.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.1.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.1.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.1.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.1.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.10.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.10.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.10.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.10.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.10.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.10.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.11.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.11.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.11.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.11.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.11.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.11.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.12.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.12.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.12.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.12.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.12.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.12.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.13.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.13.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.13.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.13.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.13.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.13.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.14.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.14.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.14.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.14.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.14.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.14.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.15.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.15.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.15.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.15.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.15.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.15.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.16.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.16.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.16.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.16.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.16.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.16.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.17.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.17.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.17.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.17.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.17.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.17.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.18.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.18.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.18.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.18.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.18.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.18.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.19.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.19.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.19.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.19.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.19.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.19.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.2.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.2.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.2.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.2.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.2.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.2.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.20.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.20.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.20.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.20.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.20.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.20.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.21.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.21.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.21.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.21.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.21.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.21.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.22.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.22.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.22.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.22.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.22.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.22.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.23.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.23.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.23.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.23.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.23.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.23.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.24.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.24.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.24.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.24.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.24.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.24.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.25.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.25.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.25.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.25.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.25.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.25.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.26.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.26.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.26.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.26.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.26.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.26.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.27.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.27.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.27.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.27.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.27.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.27.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.28.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.28.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.28.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.28.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.28.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.28.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.29.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.29.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.29.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.29.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.29.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.29.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.3.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.3.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.3.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.3.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.3.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.3.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.30.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.30.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.30.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.30.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.30.mlp.mlp_down.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.blocks.30.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.31.attn.Wqkv.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.blocks.31.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.blocks.31.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.blocks.31.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.blocks.31.mlp.mlp_down.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.blocks.31.mlp.mlp_up.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.blocks.4.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.4.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.4.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.4.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.4.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.4.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.5.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.5.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.5.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.5.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.5.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.5.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.6.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.6.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.6.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.6.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.6.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.6.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.7.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.7.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.7.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.7.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.7.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.7.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.8.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.8.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.8.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.8.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.8.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.8.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.9.attn.Wqkv.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.9.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.9.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.9.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.9.mlp.mlp_down.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.blocks.9.mlp.mlp_up.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.ln_f.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.wte.weight": "pytorch_model-00001-of-00002.bin"
+  }
+}