alex-ht
commited on
Commit
•
f767f82
1
Parent(s):
ca502f3
Update
Browse files- ultravox_processing.py +1 -1
ultravox_processing.py
CHANGED
@@ -196,7 +196,7 @@ class UltravoxProcessor(transformers.ProcessorMixin):
|
|
196 |
processed_text.append(t)
|
197 |
|
198 |
# Special tokens like BOS should already have been added by the caller.
|
199 |
-
data.update(self.tokenizer(processed_text, add_special_tokens=False, **kwargs))
|
200 |
|
201 |
return transformers.BatchFeature(data=data, tensor_type=return_tensors)
|
202 |
|
|
|
196 |
processed_text.append(t)
|
197 |
|
198 |
# Special tokens like BOS should already have been added by the caller.
|
199 |
+
data.update(self.tokenizer(processed_text, add_special_tokens=False, padding='longest', **kwargs))
|
200 |
|
201 |
return transformers.BatchFeature(data=data, tensor_type=return_tensors)
|
202 |
|