don't strip the prompt for check since we don't strip to tokenize anymore (#650)
Browse files
src/axolotl/prompt_tokenizers.py
CHANGED
@@ -81,7 +81,7 @@ class PromptTokenizingStrategy(abc.ABC):
|
|
81 |
self, prompt: str, add_eos_token: bool = True, strip_bos_token: bool = False
|
82 |
) -> BatchEncoding:
|
83 |
result: BatchEncoding
|
84 |
-
if not prompt
|
85 |
LOG.warning("Empty text requested for tokenization.")
|
86 |
result = BatchEncoding(data={"input_ids": [], "attention_mask": []})
|
87 |
else:
|
|
|
81 |
self, prompt: str, add_eos_token: bool = True, strip_bos_token: bool = False
|
82 |
) -> BatchEncoding:
|
83 |
result: BatchEncoding
|
84 |
+
if not prompt:
|
85 |
LOG.warning("Empty text requested for tokenization.")
|
86 |
result = BatchEncoding(data={"input_ids": [], "attention_mask": []})
|
87 |
else:
|