new

modeling_glm.py

@@ -763,32 +763,24 @@ class GlmModel(GlmPreTrainedModel):
             assert input_ids is not None and inputs_embeds is None, f"{input_ids} {inputs_embeds}"
             inputs_embeds = self.embed_tokens(input_ids)
             new_input_embeds = []
-            multi_flags = [True if self.config.boi_token_id in input_id.tolist() else False for input_id in input_ids]
-            images_features = None
-            if not is_empty(images):
-                images_features = self.vision(images).to(inputs_embeds.dtype)
+            boi_token_flags = [True if self.config.boi_token_id in input_id.tolist() else False for input_id in input_ids]
+            if is_empty(images):
+                images = torch.zeros([1, 3, 672, 672]).to(input_ids.device)
+            images_features = self.vision(images).to(device=inputs_embeds.device, dtype=inputs_embeds.dtype)
             image_count = 0
             for i in range(len(input_ids)):
                 input_id = input_ids[i].tolist()
-                if multi_flags[i]:
+                if boi_token_flags[i]:
                     boi_token_pos = input_id.index(self.config.boi_token_id)
                     assert boi_token_pos >= 0, "begin_of_image not found!"
                     num_image_padding_tokens = input_id.count(self.config.boi_token_id)
-                    assert (
-                        num_image_padding_tokens == images_features[image_count].shape[0]
-                    ), f"Wrong image padding token number: {num_image_padding_tokens}"
-                    new_input_embeds.append(
-                        torch.cat(
-                            (
-                                inputs_embeds[i, :boi_token_pos],
-                                images_features[image_count].to(inputs_embeds.device),
-                                inputs_embeds[i, boi_token_pos + num_image_padding_tokens :],
-                            )
-                        )
-                    )
+                    assert num_image_padding_tokens == images_features[image_count].shape[0], f"Wrong image padding token number: {num_image_padding_tokens}"
+                    new_input_embeds.append(torch.cat(
+                        (inputs_embeds[i, :boi_token_pos], images_features[image_count],
+                        inputs_embeds[i, boi_token_pos + num_image_padding_tokens:])))
                     image_count += 1
                 else:
-                    new_input_embeds.append(inputs_embeds[i])
+                    new_input_embeds.append(inputs_embeds[i] + (0 * images_features[0].sum()))
             inputs_embeds = torch.stack(new_input_embeds, dim=0)
 
         if self.gradient_checkpointing and self.training and use_cache: