prasanna kumar commited on
Commit
b9a925b
·
1 Parent(s): 3c6d585

add default values for models and shown some examples for easy try

Browse files
Files changed (1) hide show
  1. app.py +21 -1
app.py CHANGED
@@ -67,16 +67,26 @@ def process_input(input_type, input_value, model_name):
67
 
68
  return analysis, " ".join(tokens), str(token_ids), words_hist, special_chars_hist, numbers_hist
69
 
 
 
 
 
 
 
70
  with gr.Blocks() as iface:
71
  gr.Markdown("# LLM Tokenization - Convert Text to tokens and vice versa!")
72
  gr.Markdown("Enter text or token IDs and select a model to see the results, including word count, token analysis, and histograms of most common elements.")
73
 
74
  with gr.Row():
75
  input_type = gr.Radio(["Text", "Token IDs"], label="Input Type", value="Text")
76
- model_name = gr.Dropdown(choices=MODELS, label="Select Model")
77
 
78
  input_text = gr.Textbox(lines=5, label="Input")
79
 
 
 
 
 
80
  submit_button = gr.Button("Process")
81
 
82
  analysis_output = gr.Textbox(label="Analysis", lines=6)
@@ -88,6 +98,16 @@ with gr.Blocks() as iface:
88
  special_chars_plot = gr.Plot(label="Most Common Special Characters")
89
  numbers_plot = gr.Plot(label="Most Common Numbers")
90
 
 
 
 
 
 
 
 
 
 
 
91
  submit_button.click(
92
  process_input,
93
  inputs=[input_type, input_text, model_name],
 
67
 
68
  return analysis, " ".join(tokens), str(token_ids), words_hist, special_chars_hist, numbers_hist
69
 
70
+ def text_example():
71
+ return "Hello, world! This is an example text input for tokenization."
72
+
73
+ def token_ids_example():
74
+ return "[128000, 9906, 11, 1917, 0, 1115, 374, 459, 3187, 1495, 1988, 369, 4037, 2065, 13]"
75
+
76
  with gr.Blocks() as iface:
77
  gr.Markdown("# LLM Tokenization - Convert Text to tokens and vice versa!")
78
  gr.Markdown("Enter text or token IDs and select a model to see the results, including word count, token analysis, and histograms of most common elements.")
79
 
80
  with gr.Row():
81
  input_type = gr.Radio(["Text", "Token IDs"], label="Input Type", value="Text")
82
+ model_name = gr.Dropdown(choices=MODELS, label="Select Model",value=MODELS[0])
83
 
84
  input_text = gr.Textbox(lines=5, label="Input")
85
 
86
+ with gr.Row():
87
+ text_example_button = gr.Button("Load Text Example")
88
+ token_ids_example_button = gr.Button("Load Token IDs Example")
89
+
90
  submit_button = gr.Button("Process")
91
 
92
  analysis_output = gr.Textbox(label="Analysis", lines=6)
 
98
  special_chars_plot = gr.Plot(label="Most Common Special Characters")
99
  numbers_plot = gr.Plot(label="Most Common Numbers")
100
 
101
+ text_example_button.click(
102
+ lambda: (text_example(), "Text"),
103
+ outputs=[input_text, input_type]
104
+ )
105
+
106
+ token_ids_example_button.click(
107
+ lambda: (token_ids_example(), "Token IDs"),
108
+ outputs=[input_text, input_type]
109
+ )
110
+
111
  submit_button.click(
112
  process_input,
113
  inputs=[input_type, input_text, model_name],