Spaces:
Running
Running
Pclanglais
commited on
Commit
•
05775e4
1
Parent(s):
9e9c47f
Update app.py
Browse files
app.py
CHANGED
@@ -76,13 +76,13 @@ class MistralChatBot:
|
|
76 |
editorial_text = re.sub("\n", " ¶ ", user_message)
|
77 |
|
78 |
# Tokenize the prompt and check if it exceeds 500 tokens
|
79 |
-
num_tokens = len(tokenizer.tokenize(
|
80 |
|
81 |
if num_tokens > 500:
|
82 |
# Split the prompt into chunks
|
83 |
-
batch_prompts = split_text(
|
84 |
else:
|
85 |
-
batch_prompts = [
|
86 |
|
87 |
out = token_classifier(batch_prompts)
|
88 |
out = "".join(out)
|
|
|
76 |
editorial_text = re.sub("\n", " ¶ ", user_message)
|
77 |
|
78 |
# Tokenize the prompt and check if it exceeds 500 tokens
|
79 |
+
num_tokens = len(tokenizer.tokenize(editorial_text))
|
80 |
|
81 |
if num_tokens > 500:
|
82 |
# Split the prompt into chunks
|
83 |
+
batch_prompts = split_text(editorial_text, max_tokens=500)
|
84 |
else:
|
85 |
+
batch_prompts = [editorial_text]
|
86 |
|
87 |
out = token_classifier(batch_prompts)
|
88 |
out = "".join(out)
|