utrobinmv's picture
Upload tokenizer
02848ef verified
raw
history blame contribute delete
507 Bytes
{
" ": 0,
"!": 37,
"+": 40,
",": 39,
".": 36,
":": 2,
"<pad>": 42,
"<unk>": 41,
"?": 38,
"_": 1,
"а": 3,
"б": 4,
"в": 5,
"г": 6,
"д": 7,
"е": 8,
"ж": 10,
"з": 11,
"и": 12,
"й": 13,
"к": 14,
"л": 15,
"м": 16,
"н": 17,
"о": 18,
"п": 19,
"р": 20,
"с": 21,
"т": 22,
"у": 23,
"ф": 24,
"х": 25,
"ц": 26,
"ч": 27,
"ш": 28,
"щ": 29,
"ъ": 30,
"ы": 31,
"ь": 32,
"э": 33,
"ю": 34,
"я": 35,
"ё": 9
}