Upload tokenizer
deda80d
verified
|
{ |
|
"__ace_Latn__": 256002, |
|
"__ace__": 256001, |
|
"__acm__": 256003, |
|
"__acq__": 256004, |
|
"__aeb__": 256005, |
|
"__afr__": 256006, |
|
"__ajp__": 256007, |
|
"__aka__": 256008, |
|
"__als__": 256162, |
|
"__amh__": 256009, |
|
"__apc__": 256010, |
|
"__arb__": 256011, |
|
"__ars__": 256012, |
|
"__ary__": 256013, |
|
"__arz__": 256014, |
|
"__asm__": 256015, |
|
"__ast__": 256016, |
|
"__awa__": 256017, |
|
"__ayr__": 256018, |
|
"__azb__": 256019, |
|
"__azj__": 256020, |
|
"__bak__": 256021, |
|
"__bam__": 256022, |
|
"__ban__": 256023, |
|
"__bel__": 256024, |
|
"__bem__": 256025, |
|
"__ben__": 256026, |
|
"__bho__": 256027, |
|
"__bjn_Latn__": 256029, |
|
"__bjn__": 256028, |
|
"__bod__": 256030, |
|
"__bos__": 256031, |
|
"__bug__": 256032, |
|
"__bul__": 256033, |
|
"__cat__": 256034, |
|
"__ceb__": 256035, |
|
"__ces__": 256036, |
|
"__cjk__": 256037, |
|
"__ckb__": 256038, |
|
"__cmn_Hant__": 256201, |
|
"__cmn__": 256200, |
|
"__crh__": 256039, |
|
"__cym__": 256040, |
|
"__dan__": 256041, |
|
"__deu__": 256042, |
|
"__dik__": 256043, |
|
"__dyu__": 256044, |
|
"__dzo__": 256045, |
|
"__ell__": 256046, |
|
"__eng__": 256047, |
|
"__epo__": 256048, |
|
"__est__": 256049, |
|
"__eus__": 256050, |
|
"__ewe__": 256051, |
|
"__fao__": 256052, |
|
"__fij__": 256054, |
|
"__fin__": 256055, |
|
"__fon__": 256056, |
|
"__fra__": 256057, |
|
"__fur__": 256058, |
|
"__fuv__": 256059, |
|
"__gaz__": 256135, |
|
"__gla__": 256060, |
|
"__gle__": 256061, |
|
"__glg__": 256062, |
|
"__grn__": 256063, |
|
"__guj__": 256064, |
|
"__hat__": 256065, |
|
"__hau__": 256066, |
|
"__heb__": 256067, |
|
"__hin__": 256068, |
|
"__hne__": 256069, |
|
"__hrv__": 256070, |
|
"__hun__": 256071, |
|
"__hye__": 256072, |
|
"__ibo__": 256073, |
|
"__ilo__": 256074, |
|
"__ind__": 256075, |
|
"__isl__": 256076, |
|
"__ita__": 256077, |
|
"__jav__": 256078, |
|
"__jpn__": 256079, |
|
"__kab__": 256080, |
|
"__kac__": 256081, |
|
"__kam__": 256082, |
|
"__kan__": 256083, |
|
"__kas_Deva__": 256085, |
|
"__kas__": 256084, |
|
"__kat__": 256086, |
|
"__kaz__": 256089, |
|
"__kbp__": 256090, |
|
"__kea__": 256091, |
|
"__khk__": 256122, |
|
"__khm__": 256092, |
|
"__kik__": 256093, |
|
"__kin__": 256094, |
|
"__kir__": 256095, |
|
"__kmb__": 256096, |
|
"__kmr__": 256099, |
|
"__knc_Latn__": 256088, |
|
"__knc__": 256087, |
|
"__kon__": 256097, |
|
"__kor__": 256098, |
|
"__lao__": 256100, |
|
"__lij__": 256102, |
|
"__lim__": 256103, |
|
"__lin__": 256104, |
|
"__lit__": 256105, |
|
"__lmo__": 256106, |
|
"__ltg__": 256107, |
|
"__ltz__": 256108, |
|
"__lua__": 256109, |
|
"__lug__": 256110, |
|
"__luo__": 256111, |
|
"__lus__": 256112, |
|
"__lvs__": 256101, |
|
"__mag__": 256113, |
|
"__mai__": 256114, |
|
"__mal__": 256115, |
|
"__mar__": 256116, |
|
"__min__": 256117, |
|
"__mkd__": 256118, |
|
"__mlt__": 256120, |
|
"__mni__": 256121, |
|
"__mos__": 256123, |
|
"__mri__": 256124, |
|
"__mya__": 256126, |
|
"__nld__": 256127, |
|
"__nno__": 256128, |
|
"__nob__": 256129, |
|
"__npi__": 256130, |
|
"__nso__": 256131, |
|
"__nus__": 256132, |
|
"__nya__": 256133, |
|
"__oci__": 256134, |
|
"__ory__": 256136, |
|
"__pag__": 256137, |
|
"__pan__": 256138, |
|
"__pap__": 256139, |
|
"__pbt__": 256143, |
|
"__pes__": 256053, |
|
"__plt__": 256119, |
|
"__pol__": 256140, |
|
"__por__": 256141, |
|
"__prs__": 256142, |
|
"__quy__": 256144, |
|
"__ron__": 256145, |
|
"__run__": 256146, |
|
"__rus__": 256147, |
|
"__sag__": 256148, |
|
"__san__": 256149, |
|
"__sat__": 256150, |
|
"__scn__": 256151, |
|
"__shn__": 256152, |
|
"__sin__": 256153, |
|
"__slk__": 256154, |
|
"__slv__": 256155, |
|
"__smo__": 256156, |
|
"__sna__": 256157, |
|
"__snd__": 256158, |
|
"__som__": 256159, |
|
"__sot__": 256160, |
|
"__spa__": 256161, |
|
"__srd__": 256163, |
|
"__srp__": 256164, |
|
"__ssw__": 256165, |
|
"__sun__": 256166, |
|
"__swe__": 256167, |
|
"__swh__": 256168, |
|
"__szl__": 256169, |
|
"__tam__": 256170, |
|
"__taq_Tfng__": 256178, |
|
"__taq__": 256177, |
|
"__tat__": 256171, |
|
"__tel__": 256172, |
|
"__tgk__": 256173, |
|
"__tgl__": 256174, |
|
"__tha__": 256175, |
|
"__tir__": 256176, |
|
"__tpi__": 256179, |
|
"__tsn__": 256180, |
|
"__tso__": 256181, |
|
"__tuk__": 256182, |
|
"__tum__": 256183, |
|
"__tur__": 256184, |
|
"__twi__": 256185, |
|
"__tzm__": 256186, |
|
"__uig__": 256187, |
|
"__ukr__": 256188, |
|
"__umb__": 256189, |
|
"__urd__": 256190, |
|
"__uzn__": 256191, |
|
"__vec__": 256192, |
|
"__vie__": 256193, |
|
"__war__": 256194, |
|
"__wol__": 256195, |
|
"__xho__": 256196, |
|
"__ydd__": 256197, |
|
"__yor__": 256198, |
|
"__yue__": 256199, |
|
"__zsm__": 256125, |
|
"__zul__": 256202 |
|
} |
|
|