Update modeling_deepseek.py

convert dtype in ESFT so trainable experts of fp32 can be aggregated with frozen experts of bf16

modeling_deepseek.py

@@ -388,7 +388,14 @@ class DeepseekV2MLP(nn.Module):
         self.act_fn = ACT2FN[config.hidden_act]
 
     def forward(self, x):
-        down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
+        # convert dtype in ESFT so trainable experts of fp32 can be aggregated with frozen experts of bf16
+        if x.dtype != self.up_proj.weight.dtype: 
+            xdtype = x.dtype
+            x = x.to(self.up_proj.weight.dtype)
+            down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
+            down_proj = down_proj.to(xdtype)
+        else:
+            down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
         return down_proj