Spaces:
Running
Running
Update app.py
Browse files
app.py
CHANGED
@@ -11,7 +11,7 @@ def tokenize(input_text):
|
|
11 |
phi2_tokens = phi2_tokenizer(input_text, add_special_tokens=True)["input_ids"]
|
12 |
t5_tokens = t5_tokenizer(input_text, add_special_tokens=True)["input_ids"]
|
13 |
|
14 |
-
return f"LLaMa: {len(
|
15 |
|
16 |
|
17 |
if __name__ == "__main__":
|
|
|
11 |
phi2_tokens = phi2_tokenizer(input_text, add_special_tokens=True)["input_ids"]
|
12 |
t5_tokens = t5_tokenizer(input_text, add_special_tokens=True)["input_ids"]
|
13 |
|
14 |
+
return f"LLaMa: {len(llama_tokens)}\nMistral: {len(mistral_tokens)}\nGPT-2/GPT-J: {len(gpt2_tokens)}\nGPT-NeoX: {len(gpt_neox_tokens)}\nFalcon: {len(falcon_tokens)}\nPhi-2: {len(phi2_tokens)}\nT5: {len(t5_tokens)}"
|
15 |
|
16 |
|
17 |
if __name__ == "__main__":
|