Initial commit.

README.md

@@ -1,6 +1,6 @@
 ---
 title: Token Per Second Simulator
-emoji: 🔥
+emoji: ⏱️
 colorFrom: purple
 colorTo: purple
 sdk: gradio

app.py

@@ -0,0 +1,66 @@
+import gradio as gr
+import time
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1")
+
+starter_text = """# Abstract
+ Within thirty years, we will have the technological means to create superhuman intelligence. Shortly after,
+ the human era will be ended.
+ Is such progress avoidable? If not to be avoided, can events be guided so that we may survive? These questions
+ are investigated. Some possible answers (and some further dangers) are presented.
+"""
+
+
+def calculate_wait_seconds(tokens_per_second):
+    return 1 / tokens_per_second
+
+
+def get_tokens(prompt):
+    tokens = tokenizer.tokenize(prompt)
+    return [x.replace('▁', ' ').replace('<0x0A>', '\n') for x in tokens]
+
+
+def echo(message, history, prompt, tokens_per_second, time_to_first_token, stream):
+    wait_seconds = calculate_wait_seconds(tokens_per_second)
+
+    response = f"{prompt}"
+    tokens = get_tokens(response)
+
+    if time_to_first_token:
+        time.sleep(time_to_first_token / 1000)
+    partial_message = ""
+    for new_token in tokens:
+        time.sleep(wait_seconds)
+        if '<' in new_token:
+            # Gradio chat chokes on HTML-like elements
+            continue
+        partial_message += str(new_token)
+        if stream:
+            yield partial_message
+
+    if not stream:
+        yield partial_message
+
+
+with gr.Blocks(title='Tokens per Second Simulator') as demo:
+    gr.Markdown('# ⏱️ Tokens per Second Simulator')
+    gr.Markdown('Compare the feel of different response speeds for a chat bot')
+    gr.Markdown('Reading speeds vary but in English 5-10 tokens per second is considered normal reading speed')
+    gr.Markdown(
+        'References for further research:\n'
+        '- https://www.perplexity.ai/search/How-many-tokens-1d7VyXCDQuWf3pJnK4.0iw?s=c\n'
+        '- https://www.databricks.com/blog/llm-inference-performance-engineering-best-practices\n'
+        '- https://news.ycombinator.com/item?id=35978864\n'
+        '- https://www.reddit.com/r/LocalLLaMA/comments/162pgx9/what_do_yall_consider_acceptable_tokens_per/')
+
+    prompt = gr.Textbox(starter_text, label="Prompt to Echo")
+    tps_slider = gr.Slider(1, 50, render=True, value=8, label='Tokens per second (TPS)')
+    ttft_slider = gr.Slider(0, 5000, render=True, value=0,
+                            label='Time to first token (TTFT) in milliseconds')
+    stream_checkbox = gr.Checkbox(label='Stream Response', value=True)
+
+    gr.ChatInterface(echo, additional_inputs=[prompt, tps_slider, ttft_slider, stream_checkbox],
+                     description='Submit any text to echo the prompt above at the selected speed.')
+
+demo.queue().launch()