xzuyn commited on
Commit
eb70853
·
1 Parent(s): 36d9f57

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +1 -2
app.py CHANGED
@@ -5,7 +5,6 @@ import gradio as gr
5
  gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")
6
  gpt_neox_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
7
  llama_tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer")
8
- yi_tokenizer = AutoTokenizer.from_pretrained("01-ai/Yi-34B")
9
  phi2_tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-2")
10
  falcon_tokenizer = AutoTokenizer.from_pretrained("tiiuae/falcon-7b")
11
 
@@ -17,7 +16,7 @@ def tokenize(input_text):
17
  phi2_tokens = phi2_tokenizer(input_text)["input_ids"]
18
  falcon_tokens = falcon_tokenizer(input_text)["input_ids"]
19
 
20
- return f"GPT-2/GPT-J: {len(gpt2_tokens)}\nGPT-NeoX: {len(gpt_neox_tokens)}\nLLaMa: {len(llama_tokens)}\nYi: {len(yi_tokens)}\nPhi-2: {len(phi2_tokens)}\nFalcon: {len(falcon_tokens)}"
21
 
22
  iface = gr.Interface(fn=tokenize, inputs=gr.Textbox(lines=7), outputs="text")
23
  iface.launch()
 
5
  gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")
6
  gpt_neox_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
7
  llama_tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer")
 
8
  phi2_tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-2")
9
  falcon_tokenizer = AutoTokenizer.from_pretrained("tiiuae/falcon-7b")
10
 
 
16
  phi2_tokens = phi2_tokenizer(input_text)["input_ids"]
17
  falcon_tokens = falcon_tokenizer(input_text)["input_ids"]
18
 
19
+ return f"GPT-2/GPT-J: {len(gpt2_tokens)}\nGPT-NeoX: {len(gpt_neox_tokens)}\nLLaMa: {len(llama_tokens)}\nPhi-2: {len(phi2_tokens)}\nFalcon: {len(falcon_tokens)}"
20
 
21
  iface = gr.Interface(fn=tokenize, inputs=gr.Textbox(lines=7), outputs="text")
22
  iface.launch()