Upload tokenizer
3e92f84
verified
|
{ |
|
"<af>": 64002, |
|
"<am>": 64003, |
|
"<ar>": 64004, |
|
"<as>": 64005, |
|
"<az>": 64006, |
|
"<be>": 64007, |
|
"<bg>": 64008, |
|
"<bn>": 64009, |
|
"<br>": 64010, |
|
"<bs>": 64011, |
|
"<ca>": 64012, |
|
"<cs>": 64013, |
|
"<cy>": 64014, |
|
"<da>": 64015, |
|
"<de>": 64016, |
|
"<el>": 64017, |
|
"<en>": 64018, |
|
"<eo>": 64019, |
|
"<eos>": 64001, |
|
"<es>": 64020, |
|
"<et>": 64021, |
|
"<eu>": 64022, |
|
"<fa>": 64023, |
|
"<fi>": 64024, |
|
"<fr>": 64025, |
|
"<fy>": 64026, |
|
"<ga>": 64027, |
|
"<gd>": 64028, |
|
"<gl>": 64029, |
|
"<gu>": 64030, |
|
"<ha>": 64031, |
|
"<he>": 64032, |
|
"<hi>": 64033, |
|
"<hr>": 64034, |
|
"<hu>": 64035, |
|
"<id>": 64036, |
|
"<ig>": 64037, |
|
"<is>": 64038, |
|
"<it>": 64039, |
|
"<ja>": 64040, |
|
"<ka>": 64041, |
|
"<kk>": 64042, |
|
"<km>": 64043, |
|
"<kn>": 64044, |
|
"<ko>": 64045, |
|
"<ku>": 64046, |
|
"<ky>": 64047, |
|
"<li>": 64048, |
|
"<lt>": 64049, |
|
"<lv>": 64050, |
|
"<mg>": 64051, |
|
"<mk>": 64052, |
|
"<ml>": 64053, |
|
"<mr>": 64054, |
|
"<ms>": 64055, |
|
"<mt>": 64056, |
|
"<my>": 64057, |
|
"<nb>": 64058, |
|
"<ne>": 64059, |
|
"<nl>": 64060, |
|
"<nn>": 64061, |
|
"<no>": 64062, |
|
"<oc>": 64063, |
|
"<or>": 64064, |
|
"<pa>": 64065, |
|
"<pl>": 64066, |
|
"<ps>": 64067, |
|
"<pt>": 64068, |
|
"<ro>": 64069, |
|
"<ru>": 64070, |
|
"<rw>": 64071, |
|
"<se>": 64072, |
|
"<sh>": 64073, |
|
"<si>": 64074, |
|
"<sk>": 64075, |
|
"<sl>": 64076, |
|
"<sq>": 64077, |
|
"<sr>": 64078, |
|
"<sv>": 64079, |
|
"<ta>": 64080, |
|
"<te>": 64081, |
|
"<tg>": 64082, |
|
"<th>": 64083, |
|
"<tk>": 64084, |
|
"<tr>": 64085, |
|
"<tt>": 64086, |
|
"<ug>": 64087, |
|
"<uk>": 64088, |
|
"<ur>": 64089, |
|
"<uz>": 64090, |
|
"<vi>": 64091, |
|
"<wa>": 64092, |
|
"<xh>": 64093, |
|
"<yi>": 64094, |
|
"<zh>": 64095, |
|
"<zu>": 64096 |
|
} |
|
|