int4 quantized LLaVA multimodal

config.json

@@ -0,0 +1,58 @@
+{
+  "_name_or_path": "liuhaotian/LLaVA-Lightning-MPT-7B-preview",
+  "architectures": [
+    "LlavaMPTForCausalLM"
+  ],
+  "attn_config": {
+    "alibi": true,
+    "alibi_bias_max": 8,
+    "attn_impl": "torch",
+    "attn_pdrop": 0,
+    "attn_type": "multihead_attention",
+    "attn_uses_sequence_id": false,
+    "clip_qkv": null,
+    "prefix_lm": false,
+    "qk_ln": false,
+    "softmax_scale": null
+  },
+  "d_model": 4096,
+  "emb_pdrop": 0,
+  "embedding_fraction": 1.0,
+  "expansion_ratio": 4,
+  "freeze_mm_mlp_adapter": false,
+  "hidden_size": 4096,
+  "init_config": {
+    "emb_init_std": null,
+    "emb_init_uniform_lim": null,
+    "fan_mode": "fan_in",
+    "init_div_is_residual": true,
+    "init_gain": 0,
+    "init_nonlinearity": "relu",
+    "init_std": 0.02,
+    "name": "kaiming_normal_",
+    "verbose": 0
+  },
+  "init_device": "cpu",
+  "learned_pos_emb": true,
+  "logit_scale": null,
+  "max_seq_len": 2048,
+  "mm_hidden_size": 1024,
+  "mm_use_im_start_end": true,
+  "mm_vision_select_layer": -2,
+  "mm_vision_tower": "openai/clip-vit-large-patch14",
+  "model_type": "llava_mpt",
+  "n_heads": 32,
+  "n_layers": 32,
+  "no_bias": true,
+  "norm_type": "low_precision_layernorm",
+  "resid_pdrop": 0,
+  "sep_image_conv_front": false,
+  "tokenizer_name": "sam-mosaic/gpt-neox-20b-chatml",
+  "torch_dtype": "float16",
+  "transformers_version": "4.34.1",
+  "tune_mm_mlp_adapter": false,
+  "use_cache": true,
+  "use_mm_proj": true,
+  "verbose": 0,
+  "vocab_size": 50282
+}

image_encoder.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b074be38b0a337d021c092006ccd28d2554b388a6c13b3953f6760dcbf357d4
+size 296606914

llava_input_embed.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06addafc07cebdfccef95bb6d87a841f398b44d5df70b2d33a7e4e3049d5524d
+size 4223753918

llava_with_past.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c176af2f281e64725de369f710a78ed63cd660af12922a4db1199e523f531d4c
+size 4231790254

token_embed.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1203d0339c74402b0554f67768e20b5ec8274f647ff158cdca9083041907ad2
+size 411910148

token_embed.xml

@@ -0,0 +1,107 @@
+<?xml version="1.0"?>
+<net name="Model3" version="11">
+	<layers>
+		<layer id="0" name="x" type="Parameter" version="opset1">
+			<data shape="?,?" element_type="i64" />
+			<output>
+				<port id="0" precision="I64" names="x">
+					<dim>-1</dim>
+					<dim>-1</dim>
+				</port>
+			</output>
+		</layer>
+		<layer id="1" name="self.transformer.wte.weight_compressed" type="Const" version="opset1">
+			<data element_type="f16" shape="50282, 4096" offset="0" size="411910144" />
+			<output>
+				<port id="0" precision="FP16" names="4,self.transformer.wte.weight">
+					<dim>50282</dim>
+					<dim>4096</dim>
+				</port>
+			</output>
+		</layer>
+		<layer id="2" name="self.transformer.wte.weight" type="Convert" version="opset1">
+			<data destination_type="f32" />
+			<rt_info>
+				<attribute name="decompression" version="0" />
+			</rt_info>
+			<input>
+				<port id="0" precision="FP16">
+					<dim>50282</dim>
+					<dim>4096</dim>
+				</port>
+			</input>
+			<output>
+				<port id="1" precision="FP32">
+					<dim>50282</dim>
+					<dim>4096</dim>
+				</port>
+			</output>
+		</layer>
+		<layer id="3" name="__module.transformer.wte/aten::embedding/Convert" type="Convert" version="opset1">
+			<data destination_type="i32" />
+			<input>
+				<port id="0" precision="I64">
+					<dim>-1</dim>
+					<dim>-1</dim>
+				</port>
+			</input>
+			<output>
+				<port id="1" precision="I32">
+					<dim>-1</dim>
+					<dim>-1</dim>
+				</port>
+			</output>
+		</layer>
+		<layer id="4" name="__module.transformer.wte/aten::embedding/Constant" type="Const" version="opset1">
+			<data element_type="i32" shape="" offset="411910144" size="4" />
+			<output>
+				<port id="0" precision="I32" />
+			</output>
+		</layer>
+		<layer id="5" name="__module.transformer.wte/aten::embedding/Gather" type="Gather" version="opset8">
+			<data batch_dims="0" />
+			<input>
+				<port id="0" precision="FP32">
+					<dim>50282</dim>
+					<dim>4096</dim>
+				</port>
+				<port id="1" precision="I32">
+					<dim>-1</dim>
+					<dim>-1</dim>
+				</port>
+				<port id="2" precision="I32" />
+			</input>
+			<output>
+				<port id="3" precision="FP32">
+					<dim>-1</dim>
+					<dim>-1</dim>
+					<dim>4096</dim>
+				</port>
+			</output>
+		</layer>
+		<layer id="6" name="Result_48770" type="Result" version="opset1">
+			<input>
+				<port id="0" precision="FP32">
+					<dim>-1</dim>
+					<dim>-1</dim>
+					<dim>4096</dim>
+				</port>
+			</input>
+		</layer>
+	</layers>
+	<edges>
+		<edge from-layer="0" from-port="0" to-layer="3" to-port="0" />
+		<edge from-layer="1" from-port="0" to-layer="2" to-port="0" />
+		<edge from-layer="2" from-port="1" to-layer="5" to-port="0" />
+		<edge from-layer="3" from-port="1" to-layer="5" to-port="1" />
+		<edge from-layer="4" from-port="0" to-layer="5" to-port="2" />
+		<edge from-layer="5" from-port="3" to-layer="6" to-port="0" />
+	</edges>
+	<rt_info>
+		<Runtime_version value="2023.2.0-13105-ff7b49c14d1-HEAD" />
+		<conversion_parameters>
+			<framework value="pytorch" />
+			<is_python_object value="True" />
+		</conversion_parameters>
+	</rt_info>
+</net>