Update for `transformers` (#3)

- update (2bd063dce7ebecfd12724933d3c754df049a2f88)
- add pipeline (6fb8001a1f0bc0b61fac657f34f88d91a5494eb7)


Co-authored-by: Raushan Turganbay <RaushanTurganbay@users.noreply.huggingface.co>

README.md

@@ -6,7 +6,7 @@ tags:
 - image-to-text
 - image-captioning
 - visual-question-answering
-pipeline_tag: image-to-text
+pipeline_tag: image-text-to-text
 ---
 
 # BLIP-2, OPT-6.7b, fine-tuned on COCO

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<image>": 50265
+}

config.json

@@ -1,255 +1,42 @@
 {
-  "_commit_hash": null,
   "architectures": [
     "Blip2ForConditionalGeneration"
   ],
+  "image_text_hidden_size": 256,
+  "image_token_index": 50265,
   "initializer_factor": 1.0,
   "initializer_range": 0.02,
   "model_type": "blip-2",
   "num_query_tokens": 32,
   "qformer_config": {
-    "_name_or_path": "",
-    "add_cross_attention": false,
-    "architectures": null,
-    "attention_probs_dropout_prob": 0.1,
-    "bad_words_ids": null,
-    "begin_suppress_tokens": null,
-    "bos_token_id": null,
-    "chunk_size_feed_forward": 0,
     "classifier_dropout": null,
-    "cross_attention_frequency": 2,
-    "cross_attention_hidden_size": null,
-    "decoder_start_token_id": null,
-    "diversity_penalty": 0.0,
-    "do_sample": false,
-    "early_stopping": false,
-    "encoder_hidden_size": 1408,
-    "encoder_no_repeat_ngram_size": 0,
-    "eos_token_id": null,
-    "exponential_decay_length_penalty": null,
-    "finetuning_task": null,
-    "forced_bos_token_id": null,
-    "forced_eos_token_id": null,
-    "hidden_act": "gelu",
-    "hidden_dropout_prob": 0.1,
-    "hidden_size": 768,
-    "id2label": {
-      "0": "LABEL_0",
-      "1": "LABEL_1"
-    },
-    "initializer_range": 0.02,
-    "intermediate_size": 3072,
-    "is_decoder": false,
-    "is_encoder_decoder": false,
-    "label2id": {
-      "LABEL_0": 0,
-      "LABEL_1": 1
-    },
-    "layer_norm_eps": 1e-12,
-    "length_penalty": 1.0,
-    "max_length": 20,
-    "max_position_embeddings": 512,
-    "min_length": 0,
-    "model_type": "blip_2_qformer",
-    "no_repeat_ngram_size": 0,
-    "num_attention_heads": 12,
-    "num_beam_groups": 1,
-    "num_beams": 1,
-    "num_hidden_layers": 12,
-    "num_return_sequences": 1,
-    "output_attentions": false,
-    "output_hidden_states": false,
-    "output_scores": false,
-    "pad_token_id": 0,
-    "position_embedding_type": "absolute",
-    "prefix": null,
-    "problem_type": null,
-    "pruned_heads": {},
-    "remove_invalid_values": false,
-    "repetition_penalty": 1.0,
-    "return_dict": true,
-    "return_dict_in_generate": false,
-    "sep_token_id": null,
-    "suppress_tokens": null,
-    "task_specific_params": null,
-    "temperature": 1.0,
-    "tf_legacy_loss": false,
-    "tie_encoder_decoder": false,
-    "tie_word_embeddings": true,
-    "tokenizer_class": null,
-    "top_k": 50,
-    "top_p": 1.0,
-    "torch_dtype": null,
-    "torchscript": false,
-    "transformers_version": "4.27.0.dev0",
-    "typical_p": 1.0,
-    "use_bfloat16": false,
-    "vocab_size": 30522
+    "model_type": "blip_2_qformer"
   },
   "text_config": {
-    "_name_or_path": "",
-    "_remove_final_layer_norm": false,
     "activation_dropout": 0.0,
-    "activation_function": "relu",
-    "add_cross_attention": false,
     "architectures": [
       "OPTForCausalLM"
     ],
-    "attention_dropout": 0.0,
-    "bad_words_ids": null,
-    "begin_suppress_tokens": null,
-    "bos_token_id": 2,
-    "chunk_size_feed_forward": 0,
-    "cross_attention_hidden_size": null,
-    "decoder_start_token_id": null,
-    "diversity_penalty": 0.0,
-    "do_layer_norm_before": true,
-    "do_sample": false,
-    "dropout": 0.1,
-    "early_stopping": false,
-    "enable_bias": true,
-    "encoder_no_repeat_ngram_size": 0,
     "eos_token_id": 50118,
-    "exponential_decay_length_penalty": null,
     "ffn_dim": 16384,
-    "finetuning_task": null,
-    "forced_bos_token_id": null,
-    "forced_eos_token_id": null,
     "hidden_size": 4096,
-    "id2label": {
-      "0": "LABEL_0",
-      "1": "LABEL_1"
-    },
-    "init_std": 0.02,
-    "is_decoder": false,
-    "is_encoder_decoder": false,
-    "label2id": {
-      "LABEL_0": 0,
-      "LABEL_1": 1
-    },
-    "layer_norm_elementwise_affine": true,
-    "layerdrop": 0.0,
-    "length_penalty": 1.0,
-    "max_length": 20,
-    "max_position_embeddings": 2048,
-    "min_length": 0,
     "model_type": "opt",
-    "no_repeat_ngram_size": 0,
     "num_attention_heads": 32,
-    "num_beam_groups": 1,
-    "num_beams": 1,
     "num_hidden_layers": 32,
-    "num_return_sequences": 1,
-    "output_attentions": false,
-    "output_hidden_states": false,
-    "output_scores": false,
-    "pad_token_id": 1,
     "prefix": "</s>",
-    "problem_type": null,
-    "pruned_heads": {},
-    "remove_invalid_values": false,
-    "repetition_penalty": 1.0,
-    "return_dict": true,
-    "return_dict_in_generate": false,
-    "sep_token_id": null,
-    "suppress_tokens": null,
-    "task_specific_params": null,
-    "temperature": 1.0,
-    "tf_legacy_loss": false,
-    "tie_encoder_decoder": false,
-    "tie_word_embeddings": true,
-    "tokenizer_class": null,
-    "top_k": 50,
-    "top_p": 1.0,
     "torch_dtype": "float16",
-    "torchscript": false,
-    "transformers_version": "4.27.0.dev0",
-    "typical_p": 1.0,
-    "use_bfloat16": false,
-    "use_cache": true,
-    "vocab_size": 50272,
+    "vocab_size": 50304,
     "word_embed_proj_dim": 4096
   },
   "torch_dtype": "float32",
-  "transformers_version": null,
+  "transformers_version": "4.47.0.dev0",
   "use_decoder_only_language_model": true,
   "vision_config": {
-    "_name_or_path": "",
-    "add_cross_attention": false,
-    "architectures": null,
-    "attention_dropout": 0.0,
-    "bad_words_ids": null,
-    "begin_suppress_tokens": null,
-    "bos_token_id": null,
-    "chunk_size_feed_forward": 0,
-    "cross_attention_hidden_size": null,
-    "decoder_start_token_id": null,
-    "diversity_penalty": 0.0,
-    "do_sample": false,
     "dropout": 0.0,
-    "early_stopping": false,
-    "encoder_no_repeat_ngram_size": 0,
-    "eos_token_id": null,
-    "exponential_decay_length_penalty": null,
-    "finetuning_task": null,
-    "forced_bos_token_id": null,
-    "forced_eos_token_id": null,
-    "hidden_act": "gelu",
-    "hidden_size": 1408,
-    "id2label": {
-      "0": "LABEL_0",
-      "1": "LABEL_1"
-    },
     "image_size": 364,
     "initializer_factor": 1.0,
-    "initializer_range": 1e-10,
-    "intermediate_size": 6144,
-    "is_decoder": false,
-    "is_encoder_decoder": false,
-    "label2id": {
-      "LABEL_0": 0,
-      "LABEL_1": 1
-    },
-    "layer_norm_eps": 1e-6,
-    "length_penalty": 1.0,
-    "max_length": 20,
-    "min_length": 0,
     "model_type": "blip_2_vision_model",
-    "no_repeat_ngram_size": 0,
-    "num_attention_heads": 16,
-    "num_beam_groups": 1,
-    "num_beams": 1,
     "num_channels": 3,
-    "num_hidden_layers": 39,
-    "num_return_sequences": 1,
-    "output_attentions": false,
-    "output_hidden_states": false,
-    "output_scores": false,
-    "pad_token_id": null,
-    "patch_size": 14,
-    "prefix": null,
-    "problem_type": null,
-    "projection_dim": 512,
-    "pruned_heads": {},
-    "qkv_bias": true,
-    "remove_invalid_values": false,
-    "repetition_penalty": 1.0,
-    "return_dict": true,
-    "return_dict_in_generate": false,
-    "sep_token_id": null,
-    "suppress_tokens": null,
-    "task_specific_params": null,
-    "temperature": 1.0,
-    "tf_legacy_loss": false,
-    "tie_encoder_decoder": false,
-    "tie_word_embeddings": true,
-    "tokenizer_class": null,
-    "top_k": 50,
-    "top_p": 1.0,
-    "torch_dtype": null,
-    "torchscript": false,
-    "transformers_version": "4.27.0.dev0",
-    "typical_p": 1.0,
-    "use_bfloat16": false
+    "projection_dim": 512
   }
 }

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": 50118,
+  "pad_token_id": 1,
+  "transformers_version": "4.47.0.dev0"
+}

merges.txt

@@ -1,4 +1,4 @@
-#version: 0.2 - Trained by `huggingface/tokenizers`
+#version: 0.2
 Ġ t
 Ġ a
 h e

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4667d4d1adb58fb7915e8e5b39788e86c7b120edacfd98fc7bd089cccf695378
-size 9801434496
+oid sha256:550c77506bc91c75bc65c2b1d2e186c0c4a126d88fe6f69a21c322f7d37d3d14
+size 9801958784

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:54e48b80b328439e0e793252e715d1a5be5323a40b9cc6dc1bdbef8dc9e92352
-size 2166164448
+oid sha256:7e7b8eaa34f2134fe0f5d3772769d9be2158a344dc44221981d8b6dd0e80069a
+size 1342507872

processor_config.json

@@ -0,0 +1,4 @@
+{
+  "num_query_tokens": 32,
+  "processor_class": "Blip2Processor"
+}

tokenizer_config.json

@@ -1,41 +1,39 @@
 {
   "add_bos_token": true,
   "add_prefix_space": false,
-  "bos_token": {
-    "__type": "AddedToken",
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "__type": "AddedToken",
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
+  "added_tokens_decoder": {
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50265": {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
   },
+  "bos_token": "</s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
   "errors": "replace",
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": {
-    "__type": "AddedToken",
-    "content": "<pad>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
+  "pad_token": "<pad>",
   "processor_class": "Blip2Processor",
-  "special_tokens_map_file": null,
   "tokenizer_class": "GPT2Tokenizer",
-  "unk_token": {
-    "__type": "AddedToken",
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  }
+  "unk_token": "</s>"
 }