|
|
|
|
|
|
|
|
|
|
|
|
|
import os |
|
from typing import Union |
|
|
|
from transformers.configuration_utils import PretrainedConfig |
|
from transformers.utils import logging |
|
from .configuration_siglip import SiglipVisionConfig |
|
logger = logging.get_logger(__name__) |
|
|
|
|
|
class MultiBackboneChannelConcatenationVisionModelConfig(PretrainedConfig): |
|
r""" |
|
This is the configuration class to store the configuration of a [`MultiBackboneChannelConcatenationVisionModelConfig`]. It is used to |
|
instantiate a vision encoder according to the specified arguments, defining the model architecture. |
|
|
|
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the |
|
documentation from [`PretrainedConfig`] for more information. |
|
|
|
Args: |
|
vision_path (str): Path to the vision model or its configuration. |
|
mm_vision_select_layer (int, optional): The layer to select from the vision model |
|
for multi-modal processing. Defaults to -2. |
|
grid_size (int, optional): The size of the grid for vision processing. Defaults to 32. |
|
**kwargs: Additional keyword arguments to be passed to the parent PretrainedConfig. |
|
|
|
""" |
|
|
|
model_type = 'MOB' |
|
|
|
def __init__( |
|
self, |
|
vision_path, |
|
mm_vision_select_layer=-2, |
|
grid_size=32, |
|
input_image_size=1024, |
|
hidden_size='lazy_calculation', |
|
image_size=1024, |
|
freeze_backbones=None, |
|
moe_version_type=None, |
|
delay_load=False, |
|
convnext_img_size=1024, |
|
vision_tower_siglip_path=None, |
|
vision_tower_convnext_path='convnext_xxlarge.clip_laion2b_soup', |
|
normalize_type='siglip', |
|
**kwargs, |
|
): |
|
super().__init__(**kwargs) |
|
|
|
self.normalize_type = normalize_type |
|
self.vision_path = vision_path |
|
self.mm_vision_select_layer = mm_vision_select_layer |
|
self.grid_size = grid_size |
|
self.input_image_size = input_image_size |
|
self.image_size = image_size |
|
self.hidden_size = hidden_size |
|
self.freeze_backbones = freeze_backbones |
|
self.moe_version_type = moe_version_type |
|
self.delay_load = delay_load |
|
self.convnext_img_size = convnext_img_size |
|
|
|
self.vision_tower_siglip_path = vision_tower_siglip_path |
|
self.vision_tower_convnext_path = vision_tower_convnext_path |
|
self.vision_tower = self.vision_path[4:] |
|
|
|
|
|
assert image_size == input_image_size, f"input_image_size ({input_image_size}) != image_size ({image_size})" |
|
|
|
@classmethod |
|
def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> 'PretrainedConfig': |
|
config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) |
|
|
|
if 'vision_config' in config_dict: |
|
config_dict = config_dict['vision_config'] |
|
|
|
if 'model_type' in config_dict and hasattr(cls, 'model_type') and config_dict['model_type'] != cls.model_type: |
|
logger.warning( |
|
f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " |
|
f'{cls.model_type}. This is not supported for all configurations of models and can yield errors.' |
|
) |
|
|
|
return cls.from_dict(config_dict, **kwargs) |
|
|
