|
--- |
|
license: mit |
|
language: |
|
- en |
|
library_name: transformers |
|
tags: |
|
- 'vision ' |
|
- speech |
|
- image-text-text |
|
- audio-text-text |
|
- Multi-Modal |
|
--- |
|
|
|
# Creation Process |
|
<img src="https://cdn-avatars.huggingface.co/v1/production/uploads/65d883893a52cd9bcd8ab7cf/tRsCJlHNZo1D02kBTmfy9.jpeg" width="300"/> |
|
|
|
Vmodel = VisionEncoderDecoderModel.from_encoder_decoder_pretrained( |
|
"google/vit-base-patch16-224-in21k", "LeroyDyer/Mixtral_AI_Tiny" |
|
) |
|
_Encoder_ImageProcessor = Vmodel.encoder |
|
_Decoder_ImageTokenizer = Vmodel.decoder |
|
_VisionEncoderDecoderModel = Vmodel |
|
# Add Pad tokems |
|
LM_MODEL.VisionEncoderDecoder = _VisionEncoderDecoderModel |
|
# Add Sub Components |
|
LM_MODEL.Encoder_ImageProcessor = _Encoder_ImageProcessor |
|
LM_MODEL.Decoder_ImageTokenizer = _Decoder_ImageTokenizer |
|
LM_MODEL |
|
|
|
|
|
``` |
|
|
|
# ADD AUDIO |
|
|
|
```python |
|
|
|
|
|
|
|
print('Add Audio...') |
|
#Add Head |
|
# Combine pre-trained encoder and pre-trained decoder to form a Seq2Seq model |
|
_AudioFeatureExtractor = AutoFeatureExtractor.from_pretrained("openai/whisper-small") |
|
_AudioTokenizer = AutoTokenizer.from_pretrained("openai/whisper-small") |
|
_SpeechEncoderDecoder = SpeechEncoderDecoderModel.from_encoder_decoder_pretrained("openai/whisper-small","openai/whisper-small") |
|
|
|
# Add Pad tokems |
|
_SpeechEncoderDecoder.config.decoder_start_token_id = _AudioTokenizer.cls_token_id |
|
_SpeechEncoderDecoder.config.pad_token_id = _AudioTokenizer.pad_token_id |
|
LM_MODEL.SpeechEncoderDecoder = _SpeechEncoderDecoder |
|
# Add Sub Components |
|
LM_MODEL.Decoder_AudioTokenizer = _AudioTokenizer |
|
LM_MODEL.Encoder_AudioFeatureExtractor = _AudioFeatureExtractor |
|
LM_MODEL |
|
|
|
``` |
|
|
|
# SAVE |
|
```python |
|
print('Final stages:...') |
|
print('Add tokenizer...') |
|
LM_MODEL.resize_token_embeddings(len(tokenizer)) |
|
LM_MODEL.tokenizer = tokenizer |
|
print('Save model...') |
|
LM_MODEL.to(torch.float16) |
|
LM_MODEL.save_pretrained("Mixtral_AI_MiniModalTron") |
|
print('Save tokenizer...') |
|
tokenizer.save_pretrained("Mixtral_AI_MiniModalTron") |
|
|
|
``` |
