Upload model

README.md

@@ -1,7 +1,7 @@
 ---
-license: cc-by-nc-4.0
 language:
 - en
+license: cc-by-nc-4.0
 pipeline_tag: image-text-to-text
 ---
 

config.json

@@ -14,14 +14,14 @@
     "torch_dtype": "bfloat16"
   },
   "torch_dtype": "float32",
-  "transformers_version": "4.47.0",
+  "transformers_version": "4.41.2",
   "vision_encoder_config": {
     "anyres_patch_sampling": false,
     "image_aspect_ratio": "pad",
-    "model_type": "xgenmm_vision_encoder"
+    "model_type": "xgenmm_vision_encoder",
+    "temporal_encoder_mode": "gttm"
   },
   "vision_tokenizer_config": {
-    "_attn_implementation_autoset": true,
     "model_type": "xgenmm_vision_tokenizer"
   }
 }

generation_config.json

@@ -3,5 +3,5 @@
   "bos_token_id": 1,
   "eos_token_id": 32000,
   "pad_token_id": 32000,
-  "transformers_version": "4.47.0"
+  "transformers_version": "4.41.2"
 }

modeling_xgenmm.py

@@ -52,12 +52,14 @@ class XGenMMVisionTokenizerConfig(PretrainedConfig):
         lang_embedding_dim: int = 3072,
         num_vis_tokens: int = 128,
         image_aspect_ratio: str = "anyres",
+        temporal_encoder_mode: str = 'gttm',
         **kwargs,
     ):
         self.vis_feature_dim = vis_feature_dim
         self.lang_embedding_dim = lang_embedding_dim
         self.num_vis_tokens = num_vis_tokens
         self.image_aspect_ratio = image_aspect_ratio
+        self.temporal_encoder_mode = temporal_encoder_mode
         super().__init__(**kwargs)
 
 
@@ -76,6 +78,7 @@ class XGenMMConfig(PretrainedConfig):
             vision_encoder_config = {
                 "image_aspect_ratio": "pad",
                 "anyres_patch_sampling": False,
+                "temporal_encoder_mode": "gttm",
             }
             logger.info(
                 "vision_encoder_config is None. initializing the XGenMMVisionEncoderConfig with default values."
@@ -1034,7 +1037,7 @@ class PerceiverResampler(VisionTokenizer):
         max_num_media=None,
         max_num_frames=None,
         ff_mult=4,
-        video_mode='gttm',
+        temporal_encoder_mode='gttm',
     ):
         """
         Perceiver module which takes in image features and outputs image tokens.
@@ -1087,11 +1090,12 @@ class PerceiverResampler(VisionTokenizer):
 
         self.norm = nn.LayerNorm(dim)
 
-        self.video_mode = video_mode
-        if self.video_mode=='gttm':
+        self.temporal_encoder_mode = temporal_encoder_mode
+        if self.temporal_encoder_mode=='gttm':
             # self.ttm = TokenTuringMachine(dim=dim, memory_size=128, memory_out_mode=True)
             self.temporal_encoder = GroupedTokenTuringMachine(dim=dim, process_size=128, memory_size_per_group=4)
-            # self.temporal_encoder = GroupedTokenTuringMachine4(dim=dim, process_size=128, memory_size_per_group=4, output_size=32)
+        elif self.temporal_encoder_mode=='gttm_pool':
+            self.temporal_encoder = GroupedTokenTuringMachine4(dim=dim, process_size=128, memory_size_per_group=4, output_size=32)
 
     def forward(self, x, vision_attn_masks):
         """
@@ -2433,6 +2437,7 @@ class XGenMMVisionTokenizer(PreTrainedModel):
             dim_inner=config.lang_embedding_dim,
             # TODO: hardwiring for now...
             num_latents=128,
+            temporal_encoder_mode=config.temporal_encoder_mode,
         )
 
     def forward(self, vision_features: torch.Tensor, vision_attn_masks: torch.Tensor):