Jackmin108
commited on
Commit
•
7815d41
1
Parent(s):
76fc218
remove commented lines
Browse filesSigned-off-by: Meow <ongjackm@gmail.com>
block.py
CHANGED
@@ -234,16 +234,6 @@ class Block(nn.Module):
|
|
234 |
)
|
235 |
if not isinstance(self.mlp, nn.Identity):
|
236 |
mlp_out = self.mlp(hidden_states, cu_adapter_mask=mixer_kwargs.get('cu_adapter_mask'))
|
237 |
-
# if cu_adapter_mask:
|
238 |
-
# if isinstance(task_type, tuple):
|
239 |
-
# assert mixer_kwargs['cu_seqlens'].shape[0] % 9 == 1
|
240 |
-
# split_index = int((mixer_kwargs['cu_seqlens'].shape[0] - 1) / 9)
|
241 |
-
# split = mixer_kwargs['cu_seqlens'][split_index]
|
242 |
-
# mlp_out = self.mlp(hidden_states, task_type=mixer_kwargs.get('task_type'), split=split)
|
243 |
-
# else:
|
244 |
-
# mlp_out = self.mlp(hidden_states, task_type=task_type)
|
245 |
-
# else:
|
246 |
-
# mlp_out = self.mlp(hidden_states)
|
247 |
if self.return_residual: # mlp out is actually a pair here
|
248 |
mlp_out, hidden_states = mlp_out
|
249 |
if not self.fused_dropout_add_ln:
|
|
|
234 |
)
|
235 |
if not isinstance(self.mlp, nn.Identity):
|
236 |
mlp_out = self.mlp(hidden_states, cu_adapter_mask=mixer_kwargs.get('cu_adapter_mask'))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
237 |
if self.return_residual: # mlp out is actually a pair here
|
238 |
mlp_out, hidden_states = mlp_out
|
239 |
if not self.fused_dropout_add_ln:
|