Hugging Face's logo

Spaces:
concedo
/
WebTokenizer
Sleeping

App Files Community
1
xzuyn commited on
Commit
36d9f57
·
1 Parent(s): effb35b

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +1 -1
app.py CHANGED
@@ -7,7 +7,7 @@ gpt_neox_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
7
  llama_tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer")
8
  yi_tokenizer = AutoTokenizer.from_pretrained("01-ai/Yi-34B")
9
  phi2_tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-2")
10
- falcon_tokenizer = AutoTokenizer.from_pretrained("tiiuae/falcon_tokenizer")
11
 
12
  def tokenize(input_text):
13
  gpt2_tokens = gpt2_tokenizer(input_text)["input_ids"]
 
7
  llama_tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer")
8
  yi_tokenizer = AutoTokenizer.from_pretrained("01-ai/Yi-34B")
9
  phi2_tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-2")
10
+ falcon_tokenizer = AutoTokenizer.from_pretrained("tiiuae/falcon-7b")
11
 
12
  def tokenize(input_text):
13
  gpt2_tokens = gpt2_tokenizer(input_text)["input_ids"]