Hugging Face's logo

mahnerak
/
cl100k_base

Model card Files Community
cl100k_base
File size: 770 Bytes 
697f758
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
 
{
  "additional_special_tokens": [
    "<|endoftext|>",
    "<|fim_prefix|>",
    "<|fim_middle|>",
    "<|fim_suffix|>",
    "<|endofprompt|>",
    "<|begin_of_text|>",
    "<|_unuse_missing_100261|>",
    "<|_unuse_missing_100262|>",
    "<|_unuse_missing_100263|>",
    "<|_unuse_missing_100264|>",
    "<|_unuse_missing_100265|>",
    "<|_unuse_missing_100266|>",
    "<|_unuse_missing_100267|>",
    "<|_unuse_missing_100268|>",
    "<|_unuse_missing_100269|>",
    "<|_unuse_missing_100270|>",
    "<|_unuse_missing_100271|>",
    "<|_unuse_missing_100272|>",
    "<|_unuse_missing_100273|>",
    "<|_unuse_missing_100274|>",
    "<|_unuse_missing_100275|>"
  ],
  "bos_token": "<|begin_of_text|>",
  "eos_token": "<|endoftext|>",
  "unk_token": "<|endoftext|>"
}