xzuyn commited on
Commit
98985f3
1 Parent(s): a1fe124

Create app.py

Browse files
Files changed (1) hide show
  1. app.py +15 -0
app.py ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from transformers import AutoTokenizer
2
+ import gradio as gr
3
+
4
+
5
+ gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")
6
+ gptj_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-j-6b")
7
+ gpt_neox_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
8
+ llama_tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer")
9
+
10
+ def tokenize(input_text):
11
+ tokens = tokenizer(input_text)["input_ids"]
12
+ return f"Number of tokens: {len(tokens)}"
13
+
14
+ iface = gr.Interface(fn=tokenize, inputs=gr.inputs.Textbox(lines=7), outputs="text")
15
+ iface.launch()