winglian commited on
Commit
8662e8f
·
unverified ·
1 Parent(s): b2edaae

don't strip the prompt for check since we don't strip to tokenize anymore (#650)

Browse files
Files changed (1) hide show
  1. src/axolotl/prompt_tokenizers.py +1 -1
src/axolotl/prompt_tokenizers.py CHANGED
@@ -81,7 +81,7 @@ class PromptTokenizingStrategy(abc.ABC):
81
  self, prompt: str, add_eos_token: bool = True, strip_bos_token: bool = False
82
  ) -> BatchEncoding:
83
  result: BatchEncoding
84
- if not prompt.strip():
85
  LOG.warning("Empty text requested for tokenization.")
86
  result = BatchEncoding(data={"input_ids": [], "attention_mask": []})
87
  else:
 
81
  self, prompt: str, add_eos_token: bool = True, strip_bos_token: bool = False
82
  ) -> BatchEncoding:
83
  result: BatchEncoding
84
+ if not prompt:
85
  LOG.warning("Empty text requested for tokenization.")
86
  result = BatchEncoding(data={"input_ids": [], "attention_mask": []})
87
  else: