from transformers import AutoTokenizer
import gradio as gr


gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")
gptj_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-j-6b")
gpt_neox_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
llama_tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer")

def tokenize(input_text):
    tokens = tokenizer(input_text)["input_ids"]
    return f"Number of tokens: {len(tokens)}"

iface = gr.Interface(fn=tokenize, inputs=gr.inputs.Textbox(lines=7), outputs="text")
iface.launch()