Spaces:
Running
Running
from transformers import AutoTokenizer | |
import gradio as gr | |
def tokenize(input_text): | |
llama_tokens = llama2_tokenizer(input_text, add_special_tokens=True)["input_ids"] | |
mistral_tokens = mistral_tokenizer(input_text, add_special_tokens=True)["input_ids"] | |
gpt2_tokens = gpt2_tokenizer(input_text, add_special_tokens=True)["input_ids"] | |
gpt_neox_tokens = gpt_neox_tokenizer(input_text, add_special_tokens=True)["input_ids"] | |
falcon_tokens = falcon_tokenizer(input_text, add_special_tokens=True)["input_ids"] | |
phi2_tokens = phi2_tokenizer(input_text, add_special_tokens=True)["input_ids"] | |
t5_tokens = t5_tokenizer(input_text, add_special_tokens=True)["input_ids"] | |
return f"LLaMa: {len(llama1_tokens)}\nMistral: {len(mistral_tokens)}\nGPT-2/GPT-J: {len(gpt2_tokens)}\nGPT-NeoX: {len(gpt_neox_tokens)}\nFalcon: {len(falcon_tokens)}\nPhi-2: {len(phi2_tokens)}\nT5: {len(t5_tokens)}" | |
if __name__ == "__main__": | |
llama_tokenizer = AutoTokenizer.from_pretrained("TheBloke/Llama-2-7B-fp16") | |
mistral_tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1") | |
gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2") | |
gpt_neox_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b") | |
falcon_tokenizer = AutoTokenizer.from_pretrained("tiiuae/falcon-7b") | |
phi2_tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-2") | |
t5_tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-xxl") | |
iface = gr.Interface(fn=tokenize, inputs=gr.Textbox(lines=7), outputs="text") | |
iface.launch() | |