File size: 1,555 Bytes
98985f3
 
 
b5dfd52
 
b084026
275100c
39599fb
 
 
92761a6
b67636b
60a0592
b084026
98985f3
2789d18
 
b084026
e363f01
 
 
 
 
 
 
2789d18
e363f01
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
from transformers import AutoTokenizer
import gradio as gr


def tokenize(input_text):
    llama_tokens = llama2_tokenizer(input_text, add_special_tokens=True)["input_ids"]
    mistral_tokens = mistral_tokenizer(input_text, add_special_tokens=True)["input_ids"]
    gpt2_tokens = gpt2_tokenizer(input_text, add_special_tokens=True)["input_ids"]
    gpt_neox_tokens = gpt_neox_tokenizer(input_text, add_special_tokens=True)["input_ids"]
    falcon_tokens = falcon_tokenizer(input_text, add_special_tokens=True)["input_ids"]
    phi2_tokens = phi2_tokenizer(input_text, add_special_tokens=True)["input_ids"]
    t5_tokens = t5_tokenizer(input_text, add_special_tokens=True)["input_ids"]

    return f"LLaMa: {len(llama1_tokens)}\nMistral: {len(mistral_tokens)}\nGPT-2/GPT-J: {len(gpt2_tokens)}\nGPT-NeoX: {len(gpt_neox_tokens)}\nFalcon: {len(falcon_tokens)}\nPhi-2: {len(phi2_tokens)}\nT5: {len(t5_tokens)}"


if __name__ == "__main__":
    llama_tokenizer = AutoTokenizer.from_pretrained("TheBloke/Llama-2-7B-fp16")
    mistral_tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1")
    gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")
    gpt_neox_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
    falcon_tokenizer = AutoTokenizer.from_pretrained("tiiuae/falcon-7b")
    phi2_tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-2")
    t5_tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-xxl")

    iface = gr.Interface(fn=tokenize, inputs=gr.Textbox(lines=7), outputs="text")
    iface.launch()