Upload NomicBertForPreTraining

config.json

@@ -5,7 +5,7 @@
   ],
   "attn_pdrop": 0.0,
   "auto_map": {
-    "AutoConfig": "configuration_nomic_bert.NomicBertConfig",
+    "AutoConfig": "configuration_hf_nomic_bert.NomicBertConfig",
     "AutoModelForMaskedLM": "modeling_hf_nomic_bert.NomicBertForPreTraining"
   },
   "bos_token_id": null,

configuration_hf_nomic_bert.py

@@ -0,0 +1,51 @@
+from transformers import GPT2Config
+
+
+class NomicBertConfig(GPT2Config):
+    model_type = "nomic_bert"
+
+    def __init__(self,
+        prenorm=False,
+        parallel_block=False,
+        parallel_block_tied_norm=False,
+        rotary_emb_fraction=0.0,
+        fused_dropout_add_ln=False,
+        fused_bias_fc=False,
+        use_flash_attn=False,
+        use_xentropy=False,
+        qkv_proj_bias=True,
+        rotary_emb_base=1000,
+        rotary_emb_scale_base=None,
+        rotary_emb_interleaved=False,
+        mlp_fc1_bias=True,
+        mlp_fc2_bias=True,
+        use_rms_norm=False,
+        causal=False,
+        type_vocab_size=2,
+        dense_seq_output=True,
+        pad_vocab_size_multiple=1,
+        tie_word_embeddings=True,
+        **kwargs,
+    ):
+        self.prenorm = prenorm
+        self.parallel_block = parallel_block
+        self.parallel_block_tied_norm = parallel_block_tied_norm
+        self.rotary_emb_fraction = rotary_emb_fraction
+        self.tie_word_embeddings = tie_word_embeddings
+        self.fused_dropout_add_ln = fused_dropout_add_ln
+        self.fused_bias_fc = fused_bias_fc
+        self.use_flash_attn = use_flash_attn
+        self.use_xentropy = use_xentropy
+        self.qkv_proj_bias = qkv_proj_bias
+        self.rotary_emb_base = rotary_emb_base
+        self.rotary_emb_scale_base = rotary_emb_scale_base
+        self.rotary_emb_interleaved = rotary_emb_interleaved
+        self.mlp_fc1_bias = mlp_fc1_bias
+        self.mlp_fc2_bias = mlp_fc2_bias
+        self.use_rms_norm = use_rms_norm
+        self.causal = causal
+        self.type_vocab_size = type_vocab_size
+        self.dense_seq_output = dense_seq_output
+        self.pad_vocab_size_multiple = pad_vocab_size_multiple
+
+        super().__init__(**kwargs)

modeling_hf_nomic_bert.py

@@ -4,7 +4,6 @@
 # https://github.com/mlcommons/training_results_v2.1/blob/main/Azure-HazyResearch/benchmarks/bert/implementations/ND96amsr_A100_v4/modeling.py
 
 # Inspired by https://github.com/huggingface/transformers/blob/main/src/transformers/models/bert/modeling_bert.py
-
 import os
 import logging
 from functools import partial
@@ -21,12 +20,258 @@ from transformers.models.bert.modeling_bert import (
     SequenceClassifierOutput
 )
 
-from contrastors.models.encoder.configuration_nomic_bert import NomicBertConfig
-from contrastors.models.model_utils import state_dict_from_pretrained, filter_shapes
-from contrastors.models.encoder.bert import remap_bert_state_dict
+import re
+from collections import OrderedDict
+from safetensors.torch import load_file as safe_load_file
+from transformers.utils import (
+    SAFE_WEIGHTS_INDEX_NAME,
+    SAFE_WEIGHTS_NAME,
+    WEIGHTS_INDEX_NAME,
+    WEIGHTS_NAME,
+)
+from transformers.utils.hub import cached_file, get_checkpoint_shard_files
+
+
+from .configuration_hf_nomic_bert import NomicBertConfig
 
 logger = logging.getLogger(__name__)
 
+# adapted from flash attention, added safe serialization option for hf models
+def state_dict_from_pretrained(model_name, safe_serialization=False, device=None, dtype=None):
+    # If not fp32, then we don't want to load directly to the GPU
+    mapped_device = "cpu" if dtype not in [torch.float32, None] else device
+    is_sharded = False
+    load_safe = False
+    resolved_archive_file = None
+
+    weights_path = os.path.join(model_name, WEIGHTS_NAME)
+    weights_index_path = os.path.join(model_name, WEIGHTS_INDEX_NAME)
+    safe_weights_path = os.path.join(model_name, SAFE_WEIGHTS_NAME)
+    safe_weights_index_path = os.path.join(model_name, SAFE_WEIGHTS_INDEX_NAME)
+
+    if os.path.isfile(weights_path):
+        resolved_archive_file = cached_file(
+            model_name, WEIGHTS_NAME, _raise_exceptions_for_missing_entries=False
+        )
+    elif os.path.isfile(weights_index_path):
+        resolved_archive_file = cached_file(
+            model_name, WEIGHTS_INDEX_NAME, _raise_exceptions_for_missing_entries=False
+        )
+        is_sharded = True
+    elif os.path.isfile(safe_weights_path):
+        resolved_archive_file = cached_file(
+            model_name, SAFE_WEIGHTS_NAME, _raise_exceptions_for_missing_entries=False
+        )
+        load_safe = True
+    elif os.path.isfile(safe_weights_index_path):
+        resolved_archive_file = cached_file(
+            model_name, SAFE_WEIGHTS_INDEX_NAME, _raise_exceptions_for_missing_entries=False
+        )
+        is_sharded = True
+        load_safe = True
+    else:  # Try loading from HF hub instead of from local files
+        weight_name = WEIGHTS_NAME if not safe_serialization else SAFE_WEIGHTS_NAME
+        resolved_archive_file = cached_file(model_name, weight_name, _raise_exceptions_for_missing_entries=False)
+        if resolved_archive_file is None:
+            weight_index = WEIGHTS_INDEX_NAME if not safe_serialization else SAFE_WEIGHTS_INDEX_NAME
+            resolved_archive_file = cached_file(model_name, weight_index,
+                                                _raise_exceptions_for_missing_entries=False)
+            if resolved_archive_file is not None:
+                is_sharded = True
+
+        load_safe = safe_serialization
+
+    if resolved_archive_file is None:
+        raise EnvironmentError(f"Model name {model_name} was not found.")
+
+    if load_safe:
+        loader = partial(safe_load_file, device=mapped_device)
+    else:
+        loader = partial(torch.load, map_location=mapped_device)
+
+    if is_sharded:
+        # resolved_archive_file becomes a list of files that point to the different
+        # checkpoint shards in this case.
+        resolved_archive_file, sharded_metadata = get_checkpoint_shard_files(
+            model_name, resolved_archive_file
+        )
+        state_dict = {}
+        for sharded_file in resolved_archive_file:
+            state_dict.update(loader(sharded_file))
+    else:
+        state_dict = loader(resolved_archive_file)
+    # Convert dtype before moving to GPU to save memory
+    if dtype is not None:
+        state_dict = {k: v.to(dtype=dtype) for k, v in state_dict.items()}
+    state_dict = {k: v.to(device=device) for k, v in state_dict.items()}
+    return state_dict
+
+    
+def filter_shapes(state_dict, model):
+    """
+    Filters the state dict to match the current model shape.
+    """
+    filtered_state_dict = {}
+    for key, value in state_dict.items():
+        if key in model.state_dict():
+            if value.shape == model.state_dict()[key].shape:
+                filtered_state_dict[key] = value
+    return filtered_state_dict
+
+    
+def remap_bert_state_dict(state_dict, config,  remove_bert=False, remove_cls_weights=False, add_pooling_layer=False):
+    """
+    Map the state_dict of a Huggingface BERT model to be flash_attn compatible.
+    """
+    def add_bert_prefix(key):
+        # prepend bert. to the key
+        if key.startswith("bert.") or key.startswith("cls."):
+            return key
+        return f"bert.{key}"
+
+    state_dict = OrderedDict((add_bert_prefix(k), v) for k, v in state_dict.items())
+        
+    # LayerNorm
+    def key_mapping_ln_gamma_beta(key):
+        key = re.sub(r"LayerNorm.gamma$", "LayerNorm.weight", key)
+        key = re.sub(r"LayerNorm.beta$", "LayerNorm.bias", key)
+        return key
+
+    state_dict = OrderedDict((key_mapping_ln_gamma_beta(k), v) for k, v in state_dict.items())
+
+    # Layers
+    def key_mapping_layers(key):
+        return re.sub(r"^bert.encoder.layer\.", "bert.encoder.layers.", key)
+
+    state_dict = OrderedDict((key_mapping_layers(k), v) for k, v in state_dict.items())
+
+    # LayerNorm
+    def key_mapping_ln(key):
+        key = re.sub(r"^bert.embeddings.LayerNorm.", "bert.emb_ln.", key)
+        key = re.sub(
+            r"^bert.encoder.layers.(\d+).attention.output.LayerNorm.(weight|bias)",
+            r"bert.encoder.layers.\1.norm1.\2",
+            key,
+        )
+        key = re.sub(
+            r"^bert.encoder.layers.(\d+).output.LayerNorm.(weight|bias)",
+            r"bert.encoder.layers.\1.norm2.\2",
+            key,
+        )
+        key = re.sub(
+            r"^cls.predictions.transform.LayerNorm.(weight|bias)",
+            r"cls.predictions.transform.layer_norm.\1",
+            key,
+        )
+        return key
+
+    state_dict = OrderedDict((key_mapping_ln(k), v) for k, v in state_dict.items())
+
+    # MLP
+    def key_mapping_mlp(key):
+        key = re.sub(
+            r"^bert.encoder.layers.(\d+).intermediate.dense.(weight|bias)",
+            r"bert.encoder.layers.\1.mlp.fc1.\2",
+            key,
+        )
+        key = re.sub(
+            r"^bert.encoder.layers.(\d+).output.dense.(weight|bias)",
+            r"bert.encoder.layers.\1.mlp.fc2.\2",
+            key,
+        )
+        return key
+
+    state_dict = OrderedDict((key_mapping_mlp(k), v) for k, v in state_dict.items())
+
+    # Attention
+    last_layer_subset = getattr(config, "last_layer_subset", False)
+    for d in range(config.num_hidden_layers):
+        if f"bert.encoder.layers.{d}.attention.self.query.weight" not in state_dict:
+            continue
+        Wq = state_dict.pop(f"bert.encoder.layers.{d}.attention.self.query.weight")
+        Wk = state_dict.pop(f"bert.encoder.layers.{d}.attention.self.key.weight")
+        Wv = state_dict.pop(f"bert.encoder.layers.{d}.attention.self.value.weight")
+        bq = state_dict.pop(f"bert.encoder.layers.{d}.attention.self.query.bias")
+        bk = state_dict.pop(f"bert.encoder.layers.{d}.attention.self.key.bias")
+        bv = state_dict.pop(f"bert.encoder.layers.{d}.attention.self.value.bias")
+        if not (last_layer_subset and d == config.num_hidden_layers - 1):
+            state_dict[f"bert.encoder.layers.{d}.attn.Wqkv.weight"] = torch.cat(
+                [Wq, Wk, Wv], dim=0
+            )
+            state_dict[f"bert.encoder.layers.{d}.attn.Wqkv.bias"] = torch.cat([bq, bk, bv], dim=0)
+        else:
+            state_dict[f"bert.encoder.layers.{d}.attn.Wq.weight"] = Wq
+            state_dict[f"bert.encoder.layers.{d}.attn.Wkv.weight"] = torch.cat([Wk, Wv], dim=0)
+            state_dict[f"bert.encoder.layers.{d}.attn.Wq.bias"] = bq
+            state_dict[f"bert.encoder.layers.{d}.attn.Wkv.bias"] = torch.cat([bk, bv], dim=0)
+
+    def key_mapping_attn(key):
+        return re.sub(
+            r"^bert.encoder.layers.(\d+).attention.output.dense.(weight|bias)",
+            r"bert.encoder.layers.\1.attn.out_proj.\2",
+            key,
+        )
+
+    state_dict = OrderedDict((key_mapping_attn(k), v) for k, v in state_dict.items())
+
+    def key_mapping_decoder_bias(key):
+        return re.sub(r"^cls.predictions.bias", "cls.predictions.decoder.bias", key)
+
+        
+    # remove nsp weights, we don't use
+    state_dict.pop("cls.seq_relationship.weight", None)
+    state_dict.pop("cls.seq_relationship.bias", None)
+    state_dict.pop("bert.embeddings.position_ids", None)
+
+    state_dict = OrderedDict((key_mapping_decoder_bias(k), v) for k, v in state_dict.items())
+
+    # Word embedding
+    pad_vocab_size_multiple = getattr(config, "pad_vocab_size_multiple", 1)
+    if pad_vocab_size_multiple > 1:
+        word_embeddings = state_dict["bert.embeddings.word_embeddings.weight"]
+        state_dict["bert.embeddings.word_embeddings.weight"] = F.pad(
+            word_embeddings, (0, 0, 0, config.vocab_size - word_embeddings.shape[0])
+        )
+        decoder_weight = state_dict["cls.predictions.decoder.weight"]
+        state_dict["cls.predictions.decoder.weight"] = F.pad(
+            decoder_weight, (0, 0, 0, config.vocab_size - decoder_weight.shape[0])
+        )
+        # If the vocab was padded, we want to set the decoder bias for those padded indices to be
+        # strongly negative (i.e. the decoder shouldn't predict those indices).
+        # TD [2022-05-09]: I don't think it affects the MLPerf training.
+        if "cls.predictions.decoder.bias" in state_dict:
+            decoder_bias = state_dict["cls.predictions.decoder.bias"]
+            state_dict["cls.predictions.decoder.bias"] = F.pad(
+                decoder_bias, (0, config.vocab_size - decoder_bias.shape[0]), value=-100.0
+            )
+
+    if add_pooling_layer is False:
+        pooler_weights = ["bert.pooler.dense.weight",
+                          "bert.pooler.dense.bias", 
+                        ]
+        for key in pooler_weights:
+            state_dict.pop(key, None)
+
+    if remove_cls_weights:
+        cls_weights = ["cls.predictions.decoder.bias",
+                       "cls.predictions.transform.dense.weight",
+                       "cls.predictions.transform.dense.bias", 
+                       "cls.predictions.transform.layer_norm.weight", 
+                       "cls.predictions.transform.layer_norm.bias", 
+                       "cls.predictions.decoder.weight"]
+        for weight in cls_weights:
+            state_dict.pop(weight, None)
+
+    if remove_bert:
+        def remove_bert_prefix(key):
+            key = re.sub(r"^bert.", "", key)
+            return key
+
+        state_dict = OrderedDict((remove_bert_prefix(k), v) for k, v in state_dict.items())
+
+
+    return state_dict
+
 
 class NomicBertPreTrainedModel(PreTrainedModel):
     """An abstract class to handle weights initialization and