xzuyn commited on
Commit
92761a6
·
1 Parent(s): 39599fb

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +3 -4
app.py CHANGED
@@ -5,18 +5,17 @@ import gradio as gr
5
  gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")
6
  gpt_neox_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
7
  llama_tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer")
8
- phi2_tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-2")
9
  falcon_tokenizer = AutoTokenizer.from_pretrained("tiiuae/falcon-7b")
 
10
 
11
  def tokenize(input_text):
12
  gpt2_tokens = gpt2_tokenizer(input_text, add_special_tokens=True)["input_ids"]
13
  gpt_neox_tokens = gpt_neox_tokenizer(input_text, add_special_tokens=True)["input_ids"]
14
  llama_tokens = llama_tokenizer(input_text, add_special_tokens=True)["input_ids"]
15
- yi_tokens = llama_tokenizer(input_text, add_special_tokens=True)["input_ids"]
16
- phi2_tokens = phi2_tokenizer(input_text, add_special_tokens=True)["input_ids"]
17
  falcon_tokens = falcon_tokenizer(input_text, add_special_tokens=True)["input_ids"]
 
18
 
19
- return f"GPT-2/GPT-J: {len(gpt2_tokens)}\nGPT-NeoX: {len(gpt_neox_tokens)}\nLLaMa: {len(llama_tokens)}\nPhi-2: {len(phi2_tokens)}\nFalcon: {len(falcon_tokens)}"
20
 
21
  iface = gr.Interface(fn=tokenize, inputs=gr.Textbox(lines=7), outputs="text")
22
  iface.launch()
 
5
  gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")
6
  gpt_neox_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
7
  llama_tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer")
 
8
  falcon_tokenizer = AutoTokenizer.from_pretrained("tiiuae/falcon-7b")
9
+ phi2_tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-2")
10
 
11
  def tokenize(input_text):
12
  gpt2_tokens = gpt2_tokenizer(input_text, add_special_tokens=True)["input_ids"]
13
  gpt_neox_tokens = gpt_neox_tokenizer(input_text, add_special_tokens=True)["input_ids"]
14
  llama_tokens = llama_tokenizer(input_text, add_special_tokens=True)["input_ids"]
 
 
15
  falcon_tokens = falcon_tokenizer(input_text, add_special_tokens=True)["input_ids"]
16
+ phi2_tokens = phi2_tokenizer(input_text, add_special_tokens=True)["input_ids"]
17
 
18
+ return f"GPT-2/GPT-J: {len(gpt2_tokens)}\nGPT-NeoX: {len(gpt_neox_tokens)}\nLLaMa: {len(llama_tokens)}\nFalcon: {len(falcon_tokens)}\nPhi-2: {len(phi2_tokens)}"
19
 
20
  iface = gr.Interface(fn=tokenize, inputs=gr.Textbox(lines=7), outputs="text")
21
  iface.launch()