Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
@@ -43,7 +43,7 @@ def tokenize(input_text):
|
|
43 |
|
44 |
if __name__ == "__main__":
|
45 |
llama_tokenizer = AutoTokenizer.from_pretrained("TheBloke/Llama-2-7B-fp16")
|
46 |
-
llama3_tokenizer = AutoTokenizer.
|
47 |
mistral_tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1")
|
48 |
gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")
|
49 |
gpt_neox_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
|
|
|
43 |
|
44 |
if __name__ == "__main__":
|
45 |
llama_tokenizer = AutoTokenizer.from_pretrained("TheBloke/Llama-2-7B-fp16")
|
46 |
+
llama3_tokenizer = AutoTokenizer.from_pretrained("unsloth/llama-3-8b")
|
47 |
mistral_tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1")
|
48 |
gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")
|
49 |
gpt_neox_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
|