infinitejoy's picture
add tokenizer
26682d6
raw
history blame contribute delete
574 Bytes
{"'": 1, "،": 2, "؟": 3, "ء": 4, "آ": 5, "ؤ": 6, "ئ": 7, "ا": 8, "ب": 9, "ت": 10, "ث": 11, "ج": 12, "ح": 13, "خ": 14, "د": 15, "ذ": 16, "ر": 17, "ز": 18, "س": 19, "ش": 20, "ص": 21, "ض": 22, "ط": 23, "ظ": 24, "ع": 25, "غ": 26, "ف": 27, "ق": 28, "ل": 29, "م": 30, "ن": 31, "و": 32, "ى": 33, "ي": 34, "َ": 35, "ُ": 36, "ِ": 37, "ّ": 38, "ٔ": 39, "ٰ": 40, "ٹ": 41, "پ": 42, "چ": 43, "ڈ": 44, "ڑ": 45, "ژ": 46, "ک": 47, "گ": 48, "ں": 49, "ھ": 50, "ہ": 51, "ۂ": 52, "ی": 53, "ے": 54, "۔": 55, "|": 0, "[UNK]": 56, "[PAD]": 57}