Pclanglais commited on
Commit
05775e4
1 Parent(s): 9e9c47f

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +3 -3
app.py CHANGED
@@ -76,13 +76,13 @@ class MistralChatBot:
76
  editorial_text = re.sub("\n", " ¶ ", user_message)
77
 
78
  # Tokenize the prompt and check if it exceeds 500 tokens
79
- num_tokens = len(tokenizer.tokenize(prompt))
80
 
81
  if num_tokens > 500:
82
  # Split the prompt into chunks
83
- batch_prompts = split_text(prompt, max_tokens=500)
84
  else:
85
- batch_prompts = [prompt]
86
 
87
  out = token_classifier(batch_prompts)
88
  out = "".join(out)
 
76
  editorial_text = re.sub("\n", " ¶ ", user_message)
77
 
78
  # Tokenize the prompt and check if it exceeds 500 tokens
79
+ num_tokens = len(tokenizer.tokenize(editorial_text))
80
 
81
  if num_tokens > 500:
82
  # Split the prompt into chunks
83
+ batch_prompts = split_text(editorial_text, max_tokens=500)
84
  else:
85
+ batch_prompts = [editorial_text]
86
 
87
  out = token_classifier(batch_prompts)
88
  out = "".join(out)