Update convert script

convert_image_gguf.py

@@ -133,16 +133,20 @@ def main():
     fout.add_array("clip.vision.image_mean", [0.48145466, 0.4578275, 0.40821073])
     fout.add_array("clip.vision.image_std", [0.26862954, 0.26130258, 0.27577711])
 
+    fout.add_bool("clip.use_gelu", clip_vision_config["hidden_act"] != "quick_gelu")
+
     # Vision model tensors
     prefix = "model.vision_embed_tokens.img_processor.vision_model."
 
     fout.add_tensor(
         "v.class_embd",
-        tensors.get_tensor(f"{prefix}embeddings.class_embedding").astype(np.float16),
+        tensors.get_tensor(f"{prefix}embeddings.class_embedding").astype(np.float32),
     )
     fout.add_tensor(
         "v.patch_embd.weight",
-        tensors.get_tensor(f"{prefix}embeddings.patch_embedding.weight").reshape(1024, 3, 14, 14).astype(np.float16),
+        tensors.get_tensor(f"{prefix}embeddings.patch_embedding.weight")
+            .reshape(clip_vision_config["hidden_size"], 3, clip_vision_config["patch_size"], clip_vision_config["patch_size"])
+            .astype(np.float16),
     )
     fout.add_tensor(
         "v.position_embd.weight",
@@ -158,81 +162,130 @@ def main():
         tensors.get_tensor("model.vision_embed_tokens.glb_GN").astype(np.float32),
     )
 
+    fout.add_tensor(
+        "mm.0.weight",
+        tensors.get_tensor("model.vision_embed_tokens.img_projection.0.weight").astype(np.float16),
+    )
+    fout.add_tensor(
+        "mm.0.bias",
+        tensors.get_tensor("model.vision_embed_tokens.img_projection.0.bias").astype(np.float32),
+    )
+
+    fout.add_tensor(
+        "mm.2.weight",
+        tensors.get_tensor("model.vision_embed_tokens.img_projection.2.weight").astype(np.float16),
+    )
+    fout.add_tensor(
+        "mm.2.bias",
+        tensors.get_tensor("model.vision_embed_tokens.img_projection.2.bias").astype(np.float32),
+    )
+
     for i in range(clip_vision_config["num_hidden_layers"]):
-        # layer norm
+        # attention norm
         fout.add_tensor(
-            f"blk.{i}.attn_norm.weight",
+            f"v.blk.{i}.attn_norm.weight",
             tensors.get_tensor(f"{prefix}encoder.layers.{i}.layer_norm1.weight").astype(np.float32),
         )
         fout.add_tensor(
-            f"blk.{i}.attn_norm.bias",
+            f"v.blk.{i}.attn_norm.bias",
             tensors.get_tensor(f"{prefix}encoder.layers.{i}.layer_norm1.bias").astype(np.float32),
         )
         fout.add_tensor(
-            f"blk.{i}.ffn_norm.weight",
+            f"v.blk.{i}.ffn_norm.weight",
             tensors.get_tensor(f"{prefix}encoder.layers.{i}.layer_norm2.weight").astype(np.float32),
         )
         fout.add_tensor(
-            f"blk.{i}.ffn_norm.bias",
+            f"v.blk.{i}.ffn_norm.bias",
             tensors.get_tensor(f"{prefix}encoder.layers.{i}.layer_norm2.bias").astype(np.float32),
         )
 
         # feed forward
         fout.add_tensor(
-            f"blk.{i}.ffn_down.weight",
+            f"v.blk.{i}.ffn_down.weight",
             tensors.get_tensor(f"{prefix}encoder.layers.{i}.mlp.fc1.weight").astype(np.float16),
         )
         fout.add_tensor(
-            f"blk.{i}.ffn_down.bias",
-            tensors.get_tensor(f"{prefix}encoder.layers.{i}.mlp.fc1.bias").astype(np.float16),
+            f"v.blk.{i}.ffn_down.bias",
+            tensors.get_tensor(f"{prefix}encoder.layers.{i}.mlp.fc1.bias").astype(np.float32),
         )
         fout.add_tensor(
-            f"blk.{i}.ffn_up.weight",
+            f"v.blk.{i}.ffn_up.weight",
             tensors.get_tensor(f"{prefix}encoder.layers.{i}.mlp.fc2.weight").astype(np.float16),
         )
         fout.add_tensor(
-            f"blk.{i}.ffn_up.bias",
-            tensors.get_tensor(f"{prefix}encoder.layers.{i}.mlp.fc2.bias").astype(np.float16),
+            f"v.blk.{i}.ffn_up.bias",
+            tensors.get_tensor(f"{prefix}encoder.layers.{i}.mlp.fc2.bias").astype(np.float32),
         )
 
         # attention
         fout.add_tensor(
-            f"blk.{i}.attn_k.weight",
+            f"v.blk.{i}.attn_k.weight",
             tensors.get_tensor(f"{prefix}encoder.layers.{i}.self_attn.k_proj.weight").astype(np.float16),
         )
         fout.add_tensor(
-            f"blk.{i}.attn_k.bias",
-            tensors.get_tensor(f"{prefix}encoder.layers.{i}.self_attn.k_proj.bias").astype(np.float16),
+            f"v.blk.{i}.attn_k.bias",
+            tensors.get_tensor(f"{prefix}encoder.layers.{i}.self_attn.k_proj.bias").astype(np.float32),
         )
         fout.add_tensor(
-            f"blk.{i}.attn_output.weight",
+            f"v.blk.{i}.attn_out.weight",
             tensors.get_tensor(f"{prefix}encoder.layers.{i}.self_attn.out_proj.weight").astype(np.float16),
         )
         fout.add_tensor(
-            f"blk.{i}.attn_output.bias",
-            tensors.get_tensor(f"{prefix}encoder.layers.{i}.self_attn.out_proj.bias").astype(np.float16),
+            f"v.blk.{i}.attn_out.bias",
+            tensors.get_tensor(f"{prefix}encoder.layers.{i}.self_attn.out_proj.bias").astype(np.float32),
         )
         fout.add_tensor(
-            f"blk.{i}.attn_q.weight",
+            f"v.blk.{i}.attn_q.weight",
             tensors.get_tensor(f"{prefix}encoder.layers.{i}.self_attn.q_proj.weight").astype(np.float16),
         )
         fout.add_tensor(
-            f"blk.{i}.attn_q.bias",
-            tensors.get_tensor(f"{prefix}encoder.layers.{i}.self_attn.q_proj.bias").astype(np.float16),
+            f"v.blk.{i}.attn_q.bias",
+            tensors.get_tensor(f"{prefix}encoder.layers.{i}.self_attn.q_proj.bias").astype(np.float32),
         )
         fout.add_tensor(
-            f"blk.{i}.attn_v.weight",
+            f"v.blk.{i}.attn_v.weight",
             tensors.get_tensor(f"{prefix}encoder.layers.{i}.self_attn.v_proj.weight").astype(np.float16),
         )
         fout.add_tensor(
-            f"blk.{i}.attn_v.bias",
-            tensors.get_tensor(f"{prefix}encoder.layers.{i}.self_attn.v_proj.bias").astype(np.float16),
+            f"v.blk.{i}.attn_v.bias",
+            tensors.get_tensor(f"{prefix}encoder.layers.{i}.self_attn.v_proj.bias").astype(np.float32),
+        )
+
+        # layer norm
+        fout.add_tensor(
+            f"v.blk.{i}.ln1.weight",
+            tensors.get_tensor(f"{prefix}encoder.layers.{i}.layer_norm1.weight").astype(np.float32),
+        )
+        fout.add_tensor(
+            f"v.blk.{i}.ln1.bias",
+            tensors.get_tensor(f"{prefix}encoder.layers.{i}.layer_norm1.bias").astype(np.float32),
+        )
+        fout.add_tensor(
+            f"v.blk.{i}.ln2.weight",
+            tensors.get_tensor(f"{prefix}encoder.layers.{i}.layer_norm2.weight").astype(np.float32),
+        )
+        fout.add_tensor(
+            f"v.blk.{i}.ln2.bias",
+            tensors.get_tensor(f"{prefix}encoder.layers.{i}.layer_norm2.bias").astype(np.float32),
         )
 
     fout.add_tensor(
-        "output_norm.weight",
+        "v.post_ln.weight",
         tensors.get_tensor(f"{prefix}post_layernorm.weight").astype(np.float32),
     )
+    fout.add_tensor(
+        "v.post_ln.bias",
+        tensors.get_tensor(f"{prefix}post_layernorm.bias").astype(np.float32),
+    )
+
+    fout.add_tensor(
+        "v.pre_ln.weight",
+        tensors.get_tensor(f"{prefix}pre_layrnorm.weight").astype(np.float32),
+    )
+    fout.add_tensor(
+        "v.pre_ln.bias",
+        tensors.get_tensor(f"{prefix}pre_layrnorm.bias").astype(np.float32),
+    )
 
     fout.write_header_to_file()
     fout.write_kv_data_to_file()