Epoch 0. Batch Size 0. Peers 0.

config.json

@@ -1,32 +1,32 @@
 {
-  "_name_or_path": "distributed/optimized-gpt2-1b",
+  "_name_or_path": "/root/optimized-gpt2-2b",
   "activation_function": "gelu_new",
   "all_reduce_scores": {
     "0": "NON_PARTICIPATING",
     "1": "NON_PARTICIPATING",
     "10": "NON_PARTICIPATING",
     "100": "NON_PARTICIPATING",
-    "101": "SUCCESS",
+    "101": "NON_PARTICIPATING",
     "102": "NON_PARTICIPATING",
     "103": "NON_PARTICIPATING",
-    "104": "SUCCESS",
-    "105": "SUCCESS",
-    "106": "SUCCESS",
+    "104": "NON_PARTICIPATING",
+    "105": "NON_PARTICIPATING",
+    "106": "NON_PARTICIPATING",
     "107": "NON_PARTICIPATING",
     "108": "NON_PARTICIPATING",
     "109": "NON_PARTICIPATING",
-    "11": "SUCCESS",
+    "11": "NON_PARTICIPATING",
     "110": "NON_PARTICIPATING",
     "111": "NON_PARTICIPATING",
     "112": "NON_PARTICIPATING",
     "113": "NON_PARTICIPATING",
-    "114": "SUCCESS",
-    "115": "SUCCESS",
+    "114": "NON_PARTICIPATING",
+    "115": "NON_PARTICIPATING",
     "116": "NON_PARTICIPATING",
-    "117": "SUCCESS",
+    "117": "NON_PARTICIPATING",
     "118": "NON_PARTICIPATING",
-    "119": "SUCCESS",
-    "12": "SUCCESS",
+    "119": "NON_PARTICIPATING",
+    "12": "NON_PARTICIPATING",
     "120": "NON_PARTICIPATING",
     "121": "NON_PARTICIPATING",
     "122": "NON_PARTICIPATING",
@@ -37,12 +37,12 @@
     "127": "NON_PARTICIPATING",
     "128": "NON_PARTICIPATING",
     "129": "NON_PARTICIPATING",
-    "13": "SUCCESS",
+    "13": "NON_PARTICIPATING",
     "130": "NON_PARTICIPATING",
     "131": "NON_PARTICIPATING",
     "132": "NON_PARTICIPATING",
     "133": "NON_PARTICIPATING",
-    "134": "SUCCESS",
+    "134": "NON_PARTICIPATING",
     "135": "NON_PARTICIPATING",
     "136": "NON_PARTICIPATING",
     "137": "NON_PARTICIPATING",
@@ -50,9 +50,9 @@
     "139": "NON_PARTICIPATING",
     "14": "NON_PARTICIPATING",
     "140": "NON_PARTICIPATING",
-    "141": "SUCCESS",
+    "141": "NON_PARTICIPATING",
     "142": "NON_PARTICIPATING",
-    "143": "SUCCESS",
+    "143": "NON_PARTICIPATING",
     "144": "NON_PARTICIPATING",
     "145": "NON_PARTICIPATING",
     "146": "NON_PARTICIPATING",
@@ -61,16 +61,16 @@
     "149": "NON_PARTICIPATING",
     "15": "NON_PARTICIPATING",
     "150": "NON_PARTICIPATING",
-    "151": "SUCCESS",
+    "151": "NON_PARTICIPATING",
     "152": "NON_PARTICIPATING",
     "153": "NON_PARTICIPATING",
-    "154": "SUCCESS",
+    "154": "NON_PARTICIPATING",
     "155": "NON_PARTICIPATING",
-    "156": "SUCCESS",
+    "156": "NON_PARTICIPATING",
     "157": "NON_PARTICIPATING",
     "158": "NON_PARTICIPATING",
-    "159": "SUCCESS",
-    "16": "SUCCESS",
+    "159": "NON_PARTICIPATING",
+    "16": "NON_PARTICIPATING",
     "160": "NON_PARTICIPATING",
     "161": "NON_PARTICIPATING",
     "162": "NON_PARTICIPATING",
@@ -90,8 +90,8 @@
     "175": "NON_PARTICIPATING",
     "176": "NON_PARTICIPATING",
     "177": "NON_PARTICIPATING",
-    "178": "SUCCESS",
-    "179": "SUCCESS",
+    "178": "NON_PARTICIPATING",
+    "179": "NON_PARTICIPATING",
     "18": "NON_PARTICIPATING",
     "180": "NON_PARTICIPATING",
     "181": "NON_PARTICIPATING",
@@ -109,7 +109,7 @@
     "192": "NON_PARTICIPATING",
     "193": "NON_PARTICIPATING",
     "194": "NON_PARTICIPATING",
-    "195": "SUCCESS",
+    "195": "NON_PARTICIPATING",
     "196": "NON_PARTICIPATING",
     "197": "NON_PARTICIPATING",
     "198": "NON_PARTICIPATING",
@@ -117,27 +117,27 @@
     "2": "NON_PARTICIPATING",
     "20": "NON_PARTICIPATING",
     "200": "NON_PARTICIPATING",
-    "201": "SUCCESS",
-    "202": "SUCCESS",
+    "201": "NON_PARTICIPATING",
+    "202": "NON_PARTICIPATING",
     "203": "NON_PARTICIPATING",
-    "204": "SUCCESS",
+    "204": "NON_PARTICIPATING",
     "205": "NON_PARTICIPATING",
     "206": "NON_PARTICIPATING",
     "207": "NON_PARTICIPATING",
     "208": "NON_PARTICIPATING",
-    "209": "SUCCESS",
+    "209": "NON_PARTICIPATING",
     "21": "NON_PARTICIPATING",
     "210": "NON_PARTICIPATING",
     "211": "NON_PARTICIPATING",
     "212": "NON_PARTICIPATING",
-    "213": "SUCCESS",
+    "213": "NON_PARTICIPATING",
     "214": "NON_PARTICIPATING",
     "215": "NON_PARTICIPATING",
-    "216": "SUCCESS",
+    "216": "NON_PARTICIPATING",
     "217": "NON_PARTICIPATING",
     "218": "NON_PARTICIPATING",
     "219": "NON_PARTICIPATING",
-    "22": "SUCCESS",
+    "22": "NON_PARTICIPATING",
     "220": "NON_PARTICIPATING",
     "221": "NON_PARTICIPATING",
     "222": "NON_PARTICIPATING",
@@ -147,20 +147,20 @@
     "226": "NON_PARTICIPATING",
     "227": "NON_PARTICIPATING",
     "228": "NON_PARTICIPATING",
-    "229": "SUCCESS",
+    "229": "NON_PARTICIPATING",
     "23": "NON_PARTICIPATING",
     "230": "NON_PARTICIPATING",
     "231": "NON_PARTICIPATING",
     "232": "NON_PARTICIPATING",
     "233": "NON_PARTICIPATING",
     "234": "NON_PARTICIPATING",
-    "235": "SUCCESS",
-    "236": "SUCCESS",
+    "235": "NON_PARTICIPATING",
+    "236": "NON_PARTICIPATING",
     "237": "NON_PARTICIPATING",
-    "238": "SUCCESS",
-    "239": "SUCCESS",
+    "238": "NON_PARTICIPATING",
+    "239": "NON_PARTICIPATING",
     "24": "NON_PARTICIPATING",
-    "240": "SUCCESS",
+    "240": "NON_PARTICIPATING",
     "241": "NON_PARTICIPATING",
     "242": "NON_PARTICIPATING",
     "243": "NON_PARTICIPATING",
@@ -170,7 +170,7 @@
     "247": "NON_PARTICIPATING",
     "248": "NON_PARTICIPATING",
     "249": "NON_PARTICIPATING",
-    "25": "SUCCESS",
+    "25": "NON_PARTICIPATING",
     "250": "NON_PARTICIPATING",
     "251": "NON_PARTICIPATING",
     "252": "NON_PARTICIPATING",
@@ -192,71 +192,71 @@
     "37": "NON_PARTICIPATING",
     "38": "NON_PARTICIPATING",
     "39": "NON_PARTICIPATING",
-    "4": "SUCCESS",
+    "4": "NON_PARTICIPATING",
     "40": "NON_PARTICIPATING",
     "41": "NON_PARTICIPATING",
     "42": "NON_PARTICIPATING",
     "43": "NON_PARTICIPATING",
     "44": "NON_PARTICIPATING",
-    "45": "SUCCESS",
+    "45": "NON_PARTICIPATING",
     "46": "NON_PARTICIPATING",
     "47": "NON_PARTICIPATING",
-    "48": "SUCCESS",
+    "48": "NON_PARTICIPATING",
     "49": "NON_PARTICIPATING",
     "5": "NON_PARTICIPATING",
     "50": "NON_PARTICIPATING",
     "51": "NON_PARTICIPATING",
-    "52": "SUCCESS",
+    "52": "NON_PARTICIPATING",
     "53": "NON_PARTICIPATING",
     "54": "NON_PARTICIPATING",
     "55": "NON_PARTICIPATING",
-    "56": "SUCCESS",
+    "56": "NON_PARTICIPATING",
     "57": "NON_PARTICIPATING",
     "58": "NON_PARTICIPATING",
     "59": "NON_PARTICIPATING",
-    "6": "SUCCESS",
+    "6": "NON_PARTICIPATING",
     "60": "NON_PARTICIPATING",
     "61": "NON_PARTICIPATING",
-    "62": "SUCCESS",
-    "63": "SUCCESS",
+    "62": "NON_PARTICIPATING",
+    "63": "NON_PARTICIPATING",
     "64": "NON_PARTICIPATING",
-    "65": "SUCCESS",
-    "66": "SUCCESS",
+    "65": "NON_PARTICIPATING",
+    "66": "NON_PARTICIPATING",
     "67": "NON_PARTICIPATING",
     "68": "NON_PARTICIPATING",
     "69": "NON_PARTICIPATING",
-    "7": "SUCCESS",
+    "7": "NON_PARTICIPATING",
     "70": "NON_PARTICIPATING",
     "71": "NON_PARTICIPATING",
     "72": "NON_PARTICIPATING",
     "73": "NON_PARTICIPATING",
-    "74": "SUCCESS",
-    "75": "SUCCESS",
+    "74": "NON_PARTICIPATING",
+    "75": "NON_PARTICIPATING",
     "76": "NON_PARTICIPATING",
-    "77": "SUCCESS",
+    "77": "NON_PARTICIPATING",
     "78": "NON_PARTICIPATING",
     "79": "NON_PARTICIPATING",
     "8": "NON_PARTICIPATING",
-    "80": "SUCCESS",
+    "80": "NON_PARTICIPATING",
     "81": "NON_PARTICIPATING",
     "82": "NON_PARTICIPATING",
     "83": "NON_PARTICIPATING",
-    "84": "SUCCESS",
-    "85": "SUCCESS",
-    "86": "SUCCESS",
+    "84": "NON_PARTICIPATING",
+    "85": "NON_PARTICIPATING",
+    "86": "NON_PARTICIPATING",
     "87": "NON_PARTICIPATING",
-    "88": "SUCCESS",
+    "88": "NON_PARTICIPATING",
     "89": "NON_PARTICIPATING",
-    "9": "SUCCESS",
+    "9": "NON_PARTICIPATING",
     "90": "NON_PARTICIPATING",
     "91": "NON_PARTICIPATING",
     "92": "NON_PARTICIPATING",
     "93": "NON_PARTICIPATING",
     "94": "NON_PARTICIPATING",
     "95": "NON_PARTICIPATING",
-    "96": "SUCCESS",
+    "96": "NON_PARTICIPATING",
     "97": "NON_PARTICIPATING",
-    "98": "SUCCESS",
+    "98": "NON_PARTICIPATING",
     "99": "NON_PARTICIPATING"
   },
   "architectures": [
@@ -264,8 +264,8 @@
   ],
   "attn_pdrop": 0.1,
   "auto_map": {
-    "AutoConfig": "distributed/optimized-gpt2-500m--configuration_gpt_optimized.GPTOptimConfig",
-    "AutoModelForCausalLM": "distributed/optimized-gpt2-500m--modeling_gpt_optimized.GPTOptim"
+    "AutoConfig": "configuration_gpt_optimized.GPTOptimConfig",
+    "AutoModelForCausalLM": "modeling_gpt_optimized.GPTOptim"
   },
   "block_size": 1024,
   "bos_token_id": 50256,

configuration_gpt_optimized.py

@@ -0,0 +1,22 @@
+from transformers import PretrainedConfig, GPT2Config
+from typing import List
+
+
+class GPTOptimConfig(GPT2Config):
+    model_type = "gpt_optimized"
+
+    def __init__(
+        self,
+        block_size: int = 1024, # max sequence length
+        vocab_size: int = 50257, # number of tokens: 50,000 BPE merges + 256 bytes tokens + 1 <|endoftext|> token
+        n_layer: int = 16, # number of layers
+        n_head: int = 16, # number of heads
+        n_embd: int = 1024, # embedding dimension
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.block_size = block_size
+        self.vocab_size = vocab_size
+        self.n_layer = n_layer
+        self.n_head = n_head
+        self.n_embd = n_embd

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3f5ff5109f97698ac245ba95aec86c313a4f1315f0c25be9733303d0f747f8ea
-size 4975499944
+oid sha256:da60df4383c9d634d7cc53c50054860a25201fca04b7ba2222154c341c81cb64
+size 4975530824

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:82a11e07e82a54f91eddc7995fab9375336871b8701cc500ef6590e4a6f6faf4
+oid sha256:40f5cead8254114a990ff56a6870ab9b539eab67512e449d110dffba8fd755c0
 size 3075942944

model.safetensors.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 8051383872
+    "total_size": 8051414336
   },
   "weight_map": {
     "model.transformer.h.0.attn.c_attn.bias": "model-00001-of-00002.safetensors",
@@ -533,7 +533,11 @@
     "model.transformer.h.9.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
     "model.transformer.ln_f.bias": "model-00002-of-00002.safetensors",
     "model.transformer.ln_f.weight": "model-00002-of-00002.safetensors",
+    "model.transformer.wpe.norm.bias": "model-00001-of-00002.safetensors",
+    "model.transformer.wpe.norm.weight": "model-00001-of-00002.safetensors",
     "model.transformer.wpe.weight": "model-00001-of-00002.safetensors",
+    "model.transformer.wte.norm.bias": "model-00001-of-00002.safetensors",
+    "model.transformer.wte.norm.weight": "model-00001-of-00002.safetensors",
     "model.transformer.wte.weight": "model-00001-of-00002.safetensors"
   }
 }

modeling_gpt_optimized.py

@@ -0,0 +1,189 @@
+import torch
+import torch.nn as nn
+import bitsandbytes
+from torch.nn import CrossEntropyLoss, functional as F
+from transformers import PreTrainedModel, GPT2PreTrainedModel
+from .configuration_gpt_optimized import GPTOptimConfig
+from transformers.modeling_outputs import CausalLMOutputWithCrossAttentions, BaseModelOutputWithPastAndCrossAttentions
+from transformers.utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward
+from transformers.modeling_attn_mask_utils import _prepare_4d_attention_mask_for_sdpa, _prepare_4d_causal_attention_mask_for_sdpa
+from typing import Optional, Tuple, Union
+
+_CHECKPOINT_FOR_DOC = "openai-community/gpt2"
+_CONFIG_FOR_DOC = "GPT2Config"
+
+GPT2_INPUTS_DOCSTRING = r"""
+    Args:
+        input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
+            `input_ids_length` = `sequence_length` if `past_key_values` is `None` else
+            `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input
+            sequence tokens in the vocabulary.
+            If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
+            `input_ids`.
+            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+            [What are input IDs?](../glossary#input-ids)
+        past_key_values (`Tuple[Tuple[torch.Tensor]]` of length `config.n_layers`):
+            Contains precomputed hidden-states (key and values in the attention blocks) as computed by the model (see
+            `past_key_values` output below). Can be used to speed up sequential decoding. The `input_ids` which have
+            their past given to this model should not be passed as `input_ids` as they have already been computed.
+        attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+            - 1 for tokens that are **not masked**,
+            - 0 for tokens that are **masked**.
+            If `past_key_values` is used, `attention_mask` needs to contain the masking strategy that was used for
+            `past_key_values`. In other words, the `attention_mask` always has to have the length:
+            `len(past_key_values) + len(input_ids)`
+            [What are attention masks?](../glossary#attention-mask)
+        token_type_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`, *optional*):
+            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+            1]`:
+            - 0 corresponds to a *sentence A* token,
+            - 1 corresponds to a *sentence B* token.
+            [What are token type IDs?](../glossary#token-type-ids)
+        position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+            config.max_position_embeddings - 1]`.
+            [What are position IDs?](../glossary#position-ids)
+        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
+            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:
+            - 1 indicates the head is **not masked**,
+            - 0 indicates the head is **masked**.
+        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
+            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+            model's internal embedding lookup matrix.
+            If `past_key_values` is used, optionally only the last `inputs_embeds` have to be input (see
+            `past_key_values`).
+        use_cache (`bool`, *optional*):
+            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+            `past_key_values`).
+        output_attentions (`bool`, *optional*):
+            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+            tensors for more detail.
+        output_hidden_states (`bool`, *optional*):
+            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+            more detail.
+        return_dict (`bool`, *optional*):
+            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+class CausalSelfAttention(nn.Module):
+
+    def __init__(self, config):
+        super().__init__()
+        assert config.n_embd % config.n_head == 0
+        # key, query, value projections for all heads, but in a batch
+        self.c_attn = nn.Linear(config.n_embd, 3 * config.n_embd)
+        # output projection
+        self.c_proj = nn.Linear(config.n_embd, config.n_embd)
+        self.c_proj.NANOGPT_SCALE_INIT = 1
+        # regularization
+        self.n_head = config.n_head
+        self.n_embd = config.n_embd
+
+    def forward(self, x):
+        B, T, C = x.size() # batch size, sequence length, embedding dimensionality (n_embd)
+        # calculate query, key, values for all heads in batch and move head forward to be the batch dim
+        # nh is "number of heads", hs is "head size", and C (number of channels) = nh * hs
+        # e.g. in GPT-2 (124M), n_head=12, hs=64, so nh*hs=C=768 channels in the Transformer
+        qkv = self.c_attn(x)
+        q, k, v = qkv.split(self.n_embd, dim=2)
+        k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)
+        q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)
+        v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)
+        y = F.scaled_dot_product_attention(q, k, v, is_causal=True) # flash attention
+        y = y.transpose(1, 2).contiguous().view(B, T, C) # re-assemble all head outputs side by side
+        # output projection
+        y = self.c_proj(y)
+        return y
+
+class MLP(nn.Module):
+
+    def __init__(self, config):
+        super().__init__()
+        self.c_fc    = nn.Linear(config.n_embd, 4 * config.n_embd)
+        self.gelu    = nn.GELU(approximate='tanh')
+        self.c_proj  = nn.Linear(4 * config.n_embd, config.n_embd)
+        self.c_proj.NANOGPT_SCALE_INIT = 1
+
+    def forward(self, x):
+        x = self.c_fc(x)
+        x = self.gelu(x)
+        x = self.c_proj(x)
+        return x
+
+class Block(nn.Module):
+
+    def __init__(self, config):
+        super().__init__()
+        self.ln_1 = nn.LayerNorm(config.n_embd)
+        self.attn = CausalSelfAttention(config)
+        self.ln_2 = nn.LayerNorm(config.n_embd)
+        self.mlp = MLP(config)
+
+    def forward(self, x):
+        x = x + self.attn(self.ln_1(x))
+        x = x + self.mlp(self.ln_2(x))
+        return x
+
+class GPT(nn.Module):
+
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+
+        self.transformer = nn.ModuleDict(dict(
+            wte = bitsandbytes.nn.StableEmbedding(config.vocab_size, config.n_embd),
+            wpe = bitsandbytes.nn.StableEmbedding(config.block_size, config.n_embd),
+            h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
+            ln_f = nn.LayerNorm(config.n_embd),
+        ))
+        self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
+
+        # weight sharing scheme
+        self.transformer.wte.weight = self.lm_head.weight
+
+        # init params
+        self.apply(self._init_weights)
+
+    def _init_weights(self, module):
+        if isinstance(module, nn.Linear):
+            std = 0.02
+            if hasattr(module, 'NANOGPT_SCALE_INIT'):
+                std *= (2 * self.config.n_layer) ** -0.5
+            torch.nn.init.normal_(module.weight, mean=0.0, std=std)
+            if module.bias is not None:
+                torch.nn.init.zeros_(module.bias)
+        elif isinstance(module, nn.Embedding):
+            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
+
+class GPTOptim(GPT2PreTrainedModel):
+    config_class = GPTOptimConfig
+    
+    def __init__(self, config):
+        super().__init__(config)
+        self.model = GPT(
+            config
+        )
+        self.config = config
+    
+    def forward(self, input_ids, labels=None):
+        # input_ids is of shape (B, T)
+        B, T = input_ids.size()
+        assert T <= self.config.block_size, f"Cannot forward sequence of length {T}, block size is only {self.config.block_size}"
+        # forward the token and posisition embeddings
+        pos = torch.arange(0, T, dtype=torch.long, device=input_ids.device) # shape (T)
+        pos_emb = self.model.transformer.wpe(pos) # position embeddings of shape (T, n_embd)
+        tok_emb = self.model.transformer.wte(input_ids) # token embeddings of shape (B, T, n_embd)
+        x = tok_emb + pos_emb
+        # forward the blocks of the transformer
+        for block in self.model.transformer.h:
+            x = block(x)
+        # forward the final layernorm and the classifier
+        x = self.model.transformer.ln_f(x)
+        logits = self.model.lm_head(x) # (B, T, vocab_size)
+        loss = None
+        if labels is not None:
+            loss = F.cross_entropy(logits.view(-1, logits.size(-1)), labels.view(-1), ignore_index=self.config.eos_token_id)
+        return logits, loss

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c1c3e9b7f72886b3928354c660c77cc64217de83bab4aece73321c1d91dfecc
+size 2536