Upload tokenizer

tokenizer.json

@@ -1258,215 +1258,7 @@
       "kzkr": 1196,
       "Ġlcsukxlvc": 1197,
       "Ġycmne": 1198,
-      "ĠDakc": 1199,
-      "nene": 1200,
-      "upe": 1201,
-      "Ġnzkqlu": 1202,
-      "Ġblz": 1203,
-      "ĠMnc": 1204,
-      "Ġyrp": 1205,
-      "Ġlrdr": 1206,
-      "ĠDxvo": 1207,
-      "Ġnzale": 1208,
-      "ĠNzrpi": 1209,
-      "rin": 1210,
-      "Ġpepz": 1211,
-      "Ġvom": 1212,
-      "Ġtqngrlr": 1213,
-      "Ġnzkatx": 1214,
-      "opwq": 1215,
-      "ĠMenabz": 1216,
-      "ongr": 1217,
-      "pengr": 1218,
-      "Ġkzpo": 1219,
-      "ĠNepa": 1220,
-      "kaile": 1221,
-      "Ġnarlxti": 1222,
-      "ĠBoo": 1223,
-      "Ġnanginibo": 1224,
-      "Ġ1800": 1225,
-      "Ġkzpora": 1226,
-      "onrwx": 1227,
-      "Ġxlrbzkr": 1228,
-      "ĠLcsu": 1229,
-      "okr": 1230,
-      "kzamu": 1231,
-      "ĠMcnika": 1232,
-      "yrlwrpx": 1233,
-      "Ġnzyr": 1234,
-      "Ġnzlc": 1235,
-      "Ġpct": 1236,
-      "ĠTq": 1237,
-      "ĠNzlvz": 1238,
-      "Ġtrte": 1239,
-      "Ġlang": 1240,
-      "Ġnakabz": 1241,
-      "Ġayrnrngr": 1242,
-      "atele": 1243,
-      "ĠRpibo": 1244,
-      "tavetrlr": 1245,
-      "Ġtrpulu": 1246,
-      "yo": 1247,
-      "Ġnzycmne": 1248,
-      "lrla": 1249,
-      "ĠTzsiu": 1250,
-      "Ġnibrgr": 1251,
-      "ingr": 1252,
-      "Ġnzrlx": 1253,
-      "gru": 1254,
-      "Ġnzabrtz": 1255,
-      "Ġtqvzdz": 1256,
-      "kzplr": 1257,
-      "Ġnzokatr": 1258,
-      "dc": 1259,
-      "ika": 1260,
-      "ĠU": 1261,
-      "Ġnzrpwa": 1262,
-      "pxlr": 1263,
-      "Ġtqvzpem": 1264,
-      "Ġnzryrngr": 1265,
-      "galr": 1266,
-      "trpem": 1267,
-      "Ġmqde": 1268,
-      "yagoxng": 1269,
-      "yrplapx": 1270,
-      "sc": 1271,
-      "txkr": 1272,
-      "txbz": 1273,
-      "ĠJos": 1274,
-      "Ġmrb": 1275,
-      "Ġopxm": 1276,
-      "Ġkxnzetu": 1277,
-      "Ġtzmcpe": 1278,
-      "aenzli": 1279,
-      "krbzkr": 1280,
-      "lrgr": 1281,
-      "Ġbom": 1282,
-      "Ġnatwz": 1283,
-      "Ġnzsiklungr": 1284,
-      "Ġtzrtwz": 1285,
-      "Ġrpibzle": 1286,
-      "Ġnzrma": 1287,
-      "Ġlcdepwz": 1288,
-      "trpzkr": 1289,
-      "trtxpwz": 1290,
-      "ĠSi": 1291,
-      "Ġtrkrlz": 1292,
-      "Ġtutr": 1293,
-      "pxpelr": 1294,
-      "Ġtzmcpxpelr": 1295,
-      "angrti": 1296,
-      "Ġ:": 1297,
-      "ngrlvz": 1298,
-      "Ġblrkq": 1299,
-      "ĠSrpi": 1300,
-      "Ġdakc": 1301,
-      "Ġnzrtangrti": 1302,
-      "Ġdelc": 1303,
-      "son": 1304,
-      "ulz": 1305,
-      "Ġnzrp": 1306,
-      "Ġkxa": 1307,
-      "nan": 1308,
-      "rtrp": 1309,
-      "Ġnzkrlz": 1310,
-      "Ġkxglr": 1311,
-      "ngiu": 1312,
-      "ĠMznan": 1313,
-      "Ġrgrng": 1314,
-      "ĠKrus": 1315,
-      "Ġtqmcpex": 1316,
-      "ĠMznanq": 1317,
-      "prkr": 1318,
-      "wel": 1319,
-      "lrprkr": 1320,
-      "Ġseleng": 1321,
-      "bzme": 1322,
-      "Ġtrlrprkr": 1323,
-      "bom": 1324,
-      "Ġpo": 1325,
-      "mio": 1326,
-      "ĠNcnq": 1327,
-      "Ġnzrpibz": 1328,
-      "ĠSamwel": 1329,
-      "Ġtrlrprkru": 1330,
-      "ĠNcnqmz": 1331,
-      "Ġnzne": 1332,
-      "Ġncboi": 1333,
-      "obq": 1334,
-      "Ġkrla": 1335,
-      "Ġnavo": 1336,
-      "ĠNr": 1337,
-      "ĠAnika": 1338,
-      "Ġnzmwelr": 1339,
-      "opekr": 1340,
-      "Ġ?": 1341,
-      "Ġrla": 1342,
-      "Ġkcpe": 1343,
-      "nepe": 1344,
-      "ĠAtq": 1345,
-      "Ġtuti": 1346,
-      "Ġtqyrni": 1347,
-      "Ġnzrmctilr": 1348,
-      "Ġnzrlwxngr": 1349,
-      "onika": 1350,
-      "rtzlvz": 1351,
-      "Ġtqtupx": 1352,
-      "Ġtqtuom": 1353,
-      "ĠVeronika": 1354,
-      "Ġtxpwz": 1355,
-      "im": 1356,
-      "Ġtqvea": 1357,
-      "txpwzlr": 1358,
-      "ĠSamwi": 1359,
-      "Ġnzvzkilr": 1360,
-      "Ġtqdotr": 1361,
-      "Ġwiki": 1362,
-      "ĠMathi": 1363,
-      "ĠMathias": 1364,
-      "oli": 1365,
-      "Ġnzrlz": 1366,
-      "Ġtqatwzlrpe": 1367,
-      "Ġnamwi": 1368,
-      "ĠGrin": 1369,
-      "alo": 1370,
-      "Ġklas": 1371,
-      "Ġngitxpwz": 1372,
-      "iki": 1373,
-      "Ġlz": 1374,
-      "tiu": 1375,
-      "Ġtztwz": 1376,
-      "Ġtrsu": 1377,
-      "Ġkros": 1378,
-      "Ġkriki": 1379,
-      "kzx": 1380,
-      "titxpx": 1381,
-      "Ġnzrpilr": 1382,
-      "ĠBokap": 1383,
-      "Ġyzutr": 1384,
-      "Ġkzbleng": 1385,
-      "Ġtqyapxpe": 1386,
-      "Ġnzvecngr": 1387,
-      "Ġkrikit": 1388,
-      "ĠBokapwr": 1389,
-      "obzle": 1390,
-      "openg": 1391,
-      "Ġwa": 1392,
-      "Ġmnr": 1393,
-      "Ġtqtrka": 1394,
-      "lede": 1395,
-      "Ġyrb": 1396,
-      "txpxm": 1397,
-      "Ġtrbo": 1398,
-      "pin": 1399,
-      "Ġnzvzbzng": 1400,
-      "Ġkrapwx": 1401,
-      "Ġnavzkr": 1402,
-      "ĠFiu": 1403,
-      "Ġtzvzopeng": 1404,
-      "ĠTzlede": 1405,
-      "Ġtqxlr": 1406,
-      "ĠStipin": 1407
+      "ĠDakc": 1199
     },
     "merges": [
       "r t",
@@ -2586,215 +2378,7 @@
       "kz kr",
       "Ġlcsu kxlvc",
       "Ġyc mne",
-      "ĠDa kc",
-      "nen e",
-      "u pe",
-      "Ġnz kqlu",
-      "Ġb lz",
-      "ĠM nc",
-      "Ġy rp",
-      "Ġlr dr",
-      "ĠD xvo",
-      "Ġnza le",
-      "ĠNz rpi",
-      "ri n",
-      "Ġpe pz",
-      "Ġvo m",
-      "Ġtqngr lr",
-      "Ġnzka tx",
-      "op wq",
-      "ĠMen abz",
-      "o ngr",
-      "p engr",
-      "Ġkz po",
-      "ĠNe pa",
-      "kai le",
-      "Ġnarlx ti",
-      "ĠBo o",
-      "Ġnangini bo",
-      "Ġ18 00",
-      "Ġkzpo ra",
-      "onr wx",
-      "Ġxlr bzkr",
-      "ĠLc su",
-      "o kr",
-      "kz amu",
-      "ĠMc nika",
-      "yrl wrpx",
-      "Ġnz yr",
-      "Ġnz lc",
-      "Ġp ct",
-      "ĠT q",
-      "ĠNz lvz",
-      "Ġtrt e",
-      "Ġla ng",
-      "Ġnaka bz",
-      "Ġa yrnrngr",
-      "ate le",
-      "ĠRpi bo",
-      "tavetr lr",
-      "Ġtrpu lu",
-      "y o",
-      "Ġnz ycmne",
-      "lr la",
-      "ĠTz siu",
-      "Ġnibr gr",
-      "i ngr",
-      "Ġnz rlx",
-      "gr u",
-      "Ġnza brtz",
-      "Ġtqvz dz",
-      "kzp lr",
-      "Ġnzoka tr",
-      "d c",
-      "i ka",
-      "Ġ U",
-      "Ġnz rpwa",
-      "px lr",
-      "Ġtqvz pem",
-      "Ġnzr yrngr",
-      "ga lr",
-      "tr pem",
-      "Ġmq de",
-      "yagox ng",
-      "yrpla px",
-      "s c",
-      "tx kr",
-      "tx bz",
-      "ĠJ os",
-      "Ġmr b",
-      "Ġo pxm",
-      "Ġkxnz etu",
-      "Ġtzmc pe",
-      "a enzli",
-      "kr bzkr",
-      "lr gr",
-      "Ġb om",
-      "Ġna twz",
-      "Ġnzsiklu ngr",
-      "Ġtz rtwz",
-      "Ġrpi bzle",
-      "Ġnzr ma",
-      "Ġlcde pwz",
-      "trpz kr",
-      "trtx pwz",
-      "ĠS i",
-      "Ġtr krlz",
-      "Ġtu tr",
-      "pxpe lr",
-      "Ġtzmc pxpelr",
-      "a ngrti",
-      "Ġ :",
-      "ngr lvz",
-      "Ġb lrkq",
-      "ĠS rpi",
-      "Ġda kc",
-      "Ġnzrt angrti",
-      "Ġde lc",
-      "s on",
-      "u lz",
-      "Ġnz rp",
-      "Ġkx a",
-      "n an",
-      "rt rp",
-      "Ġnz krlz",
-      "Ġkx glr",
-      "ngi u",
-      "ĠMz nan",
-      "Ġrgr ng",
-      "ĠKr us",
-      "Ġtqmc pex",
-      "ĠMznan q",
-      "p rkr",
-      "w el",
-      "lr prkr",
-      "Ġs eleng",
-      "bz me",
-      "Ġtr lrprkr",
-      "bo m",
-      "Ġp o",
-      "mi o",
-      "ĠNc nq",
-      "Ġnzrpi bz",
-      "ĠSam wel",
-      "Ġtrlrprkr u",
-      "ĠNcnq mz",
-      "Ġnz ne",
-      "Ġnc boi",
-      "o bq",
-      "Ġk rla",
-      "Ġna vo",
-      "ĠN r",
-      "ĠA nika",
-      "Ġnzm welr",
-      "o pekr",
-      "Ġ ?",
-      "Ġ rla",
-      "Ġkc pe",
-      "ne pe",
-      "ĠA tq",
-      "Ġtu ti",
-      "Ġtqyr ni",
-      "Ġnzrmcti lr",
-      "Ġnzrlwx ngr",
-      "on ika",
-      "rtz lvz",
-      "Ġtqtu px",
-      "Ġtqtu om",
-      "ĠVer onika",
-      "Ġtx pwz",
-      "i m",
-      "Ġtq vea",
-      "txpwz lr",
-      "ĠSam wi",
-      "Ġnzvzki lr",
-      "Ġtqdo tr",
-      "Ġwi ki",
-      "ĠMat hi",
-      "ĠMathi as",
-      "o li",
-      "Ġnz rlz",
-      "Ġtq atwzlrpe",
-      "Ġna mwi",
-      "ĠG rin",
-      "a lo",
-      "Ġk las",
-      "Ġngi txpwz",
-      "i ki",
-      "Ġl z",
-      "ti u",
-      "Ġtz twz",
-      "Ġtr su",
-      "Ġkr os",
-      "Ġkr iki",
-      "kz x",
-      "titx px",
-      "Ġnzrpi lr",
-      "ĠBo kap",
-      "Ġyz utr",
-      "Ġkzb leng",
-      "Ġtqya pxpe",
-      "Ġnzvec ngr",
-      "Ġkriki t",
-      "ĠBokap wr",
-      "o bzle",
-      "o peng",
-      "Ġ wa",
-      "Ġm nr",
-      "Ġtq trka",
-      "le de",
-      "Ġy rb",
-      "tx pxm",
-      "Ġtr bo",
-      "pi n",
-      "Ġnzvz bzng",
-      "Ġkr apwx",
-      "Ġnavz kr",
-      "ĠF iu",
-      "Ġtzvz openg",
-      "ĠTz lede",
-      "Ġtqx lr",
-      "ĠSti pin"
+      "ĠDa kc"
     ]
   }
 }