pbt_arab_5mb / added_tokens.json
goldfish-models's picture
Upload pbt_arab_5mb tokenizer.
4e24408
raw
history blame
14.9 kB
{"[XXXXX493]": 42787, "[XXXXX219]": 42513, "[XXXXX286]": 42580, "[XXXXX392]": 42686, "[XXXXX168]": 42462, "[XXXXX547]": 42841, "[XXXXX511]": 42805, "[XXXXX55]": 42349, "[XXXXX88]": 42382, "[XXXXX47]": 42341, "[XXXXX668]": 42962, "[XXXXX401]": 42695, "[XXXXX256]": 42550, "[XXXXX533]": 42827, "[XXXXX106]": 42400, "[XXXXX569]": 42863, "[XXXXX408]": 42702, "[XXXXX564]": 42858, "[XXXXX184]": 42478, "[XXXXX181]": 42475, "[XXXXX63]": 42357, "[XXXXX550]": 42844, "[XXXXX351]": 42645, "[XXXXX161]": 42455, "[XXXXX210]": 42504, "[XXXXX582]": 42876, "[XXXXX643]": 42937, "[XXXXX323]": 42617, "[XXXXX626]": 42920, "[XXXXX576]": 42870, "[XXXXX510]": 42804, "[XXXXX527]": 42821, "[XXXXX574]": 42868, "[XXXXX311]": 42605, "[XXXXX268]": 42562, "[XXXXX15]": 42309, "[XXXXX82]": 42376, "[XXXXX399]": 42693, "[XXXXX310]": 42604, "[XXXXX30]": 42324, "[XXXXX501]": 42795, "[XXXXX330]": 42624, "[XXXXX484]": 42778, "[XXXXX209]": 42503, "[XXXXX251]": 42545, "[XXXXX153]": 42447, "[XXXXX709]": 43003, "[XXXXX205]": 42499, "[XXXXX622]": 42916, "[SEP]": 42291, "[XXXXX695]": 42989, "[XXXXX149]": 42443, "[XXXXX356]": 42650, "[XXXXX671]": 42965, "[XXXXX197]": 42491, "[XXXXX429]": 42723, "[XXXXX587]": 42881, "[XXXXX669]": 42963, "[XXXXX192]": 42486, "[XXXXX458]": 42752, "[XXXXX48]": 42342, "[XXXXX215]": 42509, "[XXXXX244]": 42538, "[XXXXX595]": 42889, "[XXXXX446]": 42740, "[XXXXX118]": 42412, "[XXXXX670]": 42964, "[XXXXX603]": 42897, "[XXXXX677]": 42971, "[XXXXX612]": 42906, "[XXXXX72]": 42366, "[XXXXX496]": 42790, "[XXXXX558]": 42852, "[XXXXX373]": 42667, "[XXXXX394]": 42688, "[XXXXX423]": 42717, "[XXXXX396]": 42690, "[XXXXX657]": 42951, "[XXXXX332]": 42626, "[XXXXX288]": 42582, "[XXXXX19]": 42313, "[XXXXX339]": 42633, "[XXXXX138]": 42432, "[XXXXX503]": 42797, "[XXXXX145]": 42439, "[XXXXX14]": 42308, "[XXXXX360]": 42654, "[XXXXX245]": 42539, "[XXXXX586]": 42880, "[XXXXX702]": 42996, "[XXXXX616]": 42910, "[XXXXX301]": 42595, "[XXXXX70]": 42364, "[XXXXX537]": 42831, "[XXXXX479]": 42773, "[XXXXX166]": 42460, "[XXXXX134]": 42428, "[XXXXX705]": 42999, "[XXXXX203]": 42497, "[XXXXX265]": 42559, "[XXXXX565]": 42859, "[XXXXX155]": 42449, "[XXXXX414]": 42708, "[XXXXX579]": 42873, "[XXXXX418]": 42712, "[XXXXX589]": 42883, "[XXXXX624]": 42918, "[XXXXX384]": 42678, "[XXXXX689]": 42983, "[XXXXX509]": 42803, "[XXXXX172]": 42466, "[XXXXX284]": 42578, "[XXXXX157]": 42451, "[XXXXX8]": 42302, "[XXXXX113]": 42407, "[XXXXX130]": 42424, "[XXXXX237]": 42531, "[XXXXX469]": 42763, "[XXXXX154]": 42448, "[XXXXX386]": 42680, "[XXXXX229]": 42523, "[XXXXX456]": 42750, "[XXXXX83]": 42377, "[XXXXX51]": 42345, "[XXXXX76]": 42370, "[XXXXX460]": 42754, "[XXXXX64]": 42358, "[XXXXX693]": 42987, "[XXXXX403]": 42697, "[XXXXX450]": 42744, "[XXXXX591]": 42885, "[XXXXX665]": 42959, "[XXXXX522]": 42816, "[XXXXX540]": 42834, "[XXXXX195]": 42489, "[XXXXX0]": 42294, "[XXXXX175]": 42469, "[XXXXX357]": 42651, "[XXXXX633]": 42927, "[XXXXX482]": 42776, "[XXXXX28]": 42322, "[XXXXX162]": 42456, "[XXXXX406]": 42700, "[XXXXX112]": 42406, "[XXXXX562]": 42856, "[XXXXX571]": 42865, "[XXXXX402]": 42696, "[XXXXX297]": 42591, "[XXXXX464]": 42758, "[XXXXX682]": 42976, "[XXXXX471]": 42765, "[XXXXX1]": 42295, "[XXXXX273]": 42567, "[XXXXX227]": 42521, "[XXXXX410]": 42704, "[XXXXX188]": 42482, "[XXXXX2]": 42296, "[XXXXX483]": 42777, "[XXXXX710]": 43004, "[XXXXX472]": 42766, "[XXXXX506]": 42800, "[XXXXX189]": 42483, "[XXXXX555]": 42849, "[XXXXX439]": 42733, "[XXXXX250]": 42544, "[XXXXX44]": 42338, "[XXXXX77]": 42371, "[XXXXX352]": 42646, "[XXXXX468]": 42762, "[XXXXX193]": 42487, "[XXXXX92]": 42386, "[XXXXX207]": 42501, "[XXXXX494]": 42788, "[XXXXX58]": 42352, "[XXXXX615]": 42909, "[XXXXX620]": 42914, "[XXXXX187]": 42481, "[XXXXX542]": 42836, "[XXXXX235]": 42529, "[XXXXX9]": 42303, "[XXXXX432]": 42726, "[XXXXX376]": 42670, "[XXXXX290]": 42584, "[XXXXX391]": 42685, "[XXXXX395]": 42689, "[XXXXX125]": 42419, "[XXXXX630]": 42924, "[XXXXX640]": 42934, "[XXXXX194]": 42488, "[XXXXX53]": 42347, "[XXXXX552]": 42846, "[XXXXX578]": 42872, "[XXXXX516]": 42810, "[XXXXX171]": 42465, "[XXXXX635]": 42929, "[XXXXX186]": 42480, "[XXXXX597]": 42891, "[XXXXX32]": 42326, "[XXXXX208]": 42502, "[XXXXX634]": 42928, "[XXXXX457]": 42751, "[XXXXX398]": 42692, "[XXXXX713]": 43007, "[XXXXX355]": 42649, "[XXXXX534]": 42828, "[XXXXX529]": 42823, "[XXXXX672]": 42966, "[XXXXX212]": 42506, "[XXXXX6]": 42300, "[XXXXX254]": 42548, "[XXXXX327]": 42621, "[XXXXX608]": 42902, "[XXXXX649]": 42943, "[XXXXX272]": 42566, "[XXXXX128]": 42422, "[XXXXX430]": 42724, "[XXXXX378]": 42672, "[XXXXX690]": 42984, "[XXXXX328]": 42622, "[XXXXX644]": 42938, "[XXXXX299]": 42593, "[XXXXX476]": 42770, "[XXXXX216]": 42510, "[XXXXX413]": 42707, "[XXXXX220]": 42514, "[XXXXX681]": 42975, "[XXXXX359]": 42653, "[XXXXX452]": 42746, "[XXXXX75]": 42369, "[XXXXX283]": 42577, "[XXXXX585]": 42879, "[XXXXX340]": 42634, "[XXXXX223]": 42517, "[XXXXX521]": 42815, "[XXXXX557]": 42851, "[XXXXX269]": 42563, "[XXXXX545]": 42839, "[XXXXX687]": 42981, "[XXXXX602]": 42896, "[XXXXX454]": 42748, "[XXXXX241]": 42535, "[XXXXX448]": 42742, "[XXXXX321]": 42615, "[XXXXX338]": 42632, "[XXXXX703]": 42997, "[XXXXX263]": 42557, "[XXXXX353]": 42647, "[XXXXX380]": 42674, "[XXXXX632]": 42926, "[XXXXX226]": 42520, "[XXXXX504]": 42798, "[XXXXX139]": 42433, "[XXXXX563]": 42857, "[XXXXX646]": 42940, "[XXXXX344]": 42638, "[XXXXX326]": 42620, "[XXXXX377]": 42671, "[XXXXX11]": 42305, "[XXXXX524]": 42818, "[XXXXX528]": 42822, "[XXXXX68]": 42362, "[XXXXX631]": 42925, "[XXXXX618]": 42912, "[XXXXX374]": 42668, "[XXXXX278]": 42572, "[XXXXX233]": 42527, "[XXXXX428]": 42722, "[XXXXX478]": 42772, "[XXXXX182]": 42476, "[XXXXX389]": 42683, "[XXXXX306]": 42600, "[XXXXX364]": 42658, "[XXXXX696]": 42990, "[XXXXX176]": 42470, "[XXXXX686]": 42980, "[XXXXX685]": 42979, "[XXXXX117]": 42411, "[XXXXX25]": 42319, "[XXXXX614]": 42908, "[XXXXX36]": 42330, "[XXXXX238]": 42532, "[XXXXX387]": 42681, "[XXXXX502]": 42796, "[XXXXX271]": 42565, "[XXXXX313]": 42607, "[XXXXX170]": 42464, "[XXXXX102]": 42396, "[XXXXX62]": 42356, "[XXXXX652]": 42946, "[XXXXX451]": 42745, "[XXXXX538]": 42832, "[XXXXX336]": 42630, "[XXXXX449]": 42743, "[XXXXX663]": 42957, "[XXXXX701]": 42995, "[XXXXX276]": 42570, "[XXXXX661]": 42955, "[XXXXX107]": 42401, "[XXXXX247]": 42541, "[XXXXX74]": 42368, "[XXXXX514]": 42808, "[XXXXX126]": 42420, "[XXXXX218]": 42512, "[XXXXX3]": 42297, "[XXXXX21]": 42315, "[XXXXX607]": 42901, "[XXXXX89]": 42383, "[XXXXX291]": 42585, "[XXXXX711]": 43005, "[XXXXX645]": 42939, "[XXXXX329]": 42623, "[XXXXX56]": 42350, "[XXXXX699]": 42993, "[XXXXX666]": 42960, "[XXXXX400]": 42694, "[XXXXX680]": 42974, "[XXXXX427]": 42721, "[XXXXX470]": 42764, "[XXXXX700]": 42994, "[XXXXX13]": 42307, "[XXXXX531]": 42825, "[XXXXX580]": 42874, "[XXXXX99]": 42393, "[XXXXX177]": 42471, "[XXXXX343]": 42637, "[XXXXX366]": 42660, "[XXXXX133]": 42427, "[XXXXX300]": 42594, "[XXXXX583]": 42877, "[XXXXX240]": 42534, "[XXXXX27]": 42321, "[XXXXX158]": 42452, "[XXXXX354]": 42648, "[XXXXX660]": 42954, "[XXXXX180]": 42474, "[XXXXX539]": 42833, "[XXXXX50]": 42344, "[XXXXX293]": 42587, "[XXXXX178]": 42472, "[XXXXX191]": 42485, "[XXXXX201]": 42495, "[XXXXX697]": 42991, "[XXXXX101]": 42395, "[XXXXX447]": 42741, "[XXXXX592]": 42886, "[XXXXX590]": 42884, "[XXXXX481]": 42775, "[XXXXX708]": 43002, "[XXXXX556]": 42850, "[MASK]": 42293, "[XXXXX304]": 42598, "[XXXXX426]": 42720, "[XXXXX206]": 42500, "[XXXXX81]": 42375, "[XXXXX33]": 42327, "[XXXXX42]": 42336, "[XXXXX294]": 42588, "[XXXXX66]": 42360, "[XXXXX320]": 42614, "[XXXXX637]": 42931, "[XXXXX434]": 42728, "[XXXXX211]": 42505, "[XXXXX656]": 42950, "[XXXXX422]": 42716, "[XXXXX621]": 42915, "[XXXXX22]": 42316, "[XXXXX248]": 42542, "[XXXXX658]": 42952, "[XXXXX362]": 42656, "[XXXXX605]": 42899, "[XXXXX69]": 42363, "[XXXXX122]": 42416, "[XXXXX267]": 42561, "[XXXXX536]": 42830, "[XXXXX249]": 42543, "[XXXXX164]": 42458, "[CLS]": 42290, "[XXXXX335]": 42629, "[XXXXX495]": 42789, "[XXXXX367]": 42661, "[XXXXX305]": 42599, "[XXXXX146]": 42440, "[XXXXX487]": 42781, "[XXXXX415]": 42709, "[XXXXX264]": 42558, "[XXXXX34]": 42328, "[XXXXX674]": 42968, "[XXXXX706]": 43000, "[XXXXX46]": 42340, "[XXXXX324]": 42618, "[XXXXX431]": 42725, "[XXXXX676]": 42970, "[XXXXX610]": 42904, "[XXXXX370]": 42664, "[XXXXX517]": 42811, "[XXXXX601]": 42895, "[XXXXX594]": 42888, "[XXXXX179]": 42473, "[XXXXX234]": 42528, "[XXXXX39]": 42333, "[XXXXX498]": 42792, "[XXXXX480]": 42774, "[XXXXX416]": 42710, "[XXXXX140]": 42434, "[XXXXX548]": 42842, "[XXXXX347]": 42641, "[XXXXX691]": 42985, "[XXXXX49]": 42343, "[XXXXX417]": 42711, "[XXXXX554]": 42848, "[XXXXX142]": 42436, "[XXXXX150]": 42444, "[XXXXX553]": 42847, "[XXXXX91]": 42385, "[XXXXX52]": 42346, "[XXXXX419]": 42713, "[XXXXX204]": 42498, "[XXXXX477]": 42771, "[XXXXX683]": 42977, "[XXXXX345]": 42639, "[XXXXX224]": 42518, "[XXXXX393]": 42687, "[XXXXX513]": 42807, "[XXXXX559]": 42853, "[XXXXX596]": 42890, "[XXXXX371]": 42665, "[XXXXX243]": 42537, "[XXXXX325]": 42619, "[XXXXX628]": 42922, "[XXXXX348]": 42642, "[XXXXX575]": 42869, "[XXXXX147]": 42441, "[XXXXX461]": 42755, "[XXXXX59]": 42353, "[XXXXX684]": 42978, "[XXXXX525]": 42819, "[XXXXX123]": 42417, "[XXXXX287]": 42581, "[XXXXX131]": 42425, "[XXXXX566]": 42860, "[XXXXX341]": 42635, "[XXXXX26]": 42320, "[XXXXX196]": 42490, "[XXXXX57]": 42351, "[XXXXX309]": 42603, "[XXXXX67]": 42361, "[XXXXX221]": 42515, "[XXXXX135]": 42429, "[XXXXX35]": 42329, "[XXXXX629]": 42923, "[XXXXX490]": 42784, "[XXXXX119]": 42413, "[XXXXX598]": 42892, "[XXXXX593]": 42887, "[XXXXX144]": 42438, "[XXXXX137]": 42431, "[XXXXX441]": 42735, "[XXXXX606]": 42900, "[XXXXX275]": 42569, "[XXXXX437]": 42731, "[XXXXX261]": 42555, "[XXXXX183]": 42477, "[XXXXX316]": 42610, "[XXXXX425]": 42719, "[XXXXX280]": 42574, "[XXXXX174]": 42468, "[XXXXX7]": 42301, "[XXXXX334]": 42628, "[XXXXX156]": 42450, "[XXXXX136]": 42430, "[XXXXX349]": 42643, "[XXXXX397]": 42691, "[XXXXX317]": 42611, "[XXXXX292]": 42586, "[XXXXX420]": 42714, "[XXXXX694]": 42988, "[XXXXX486]": 42780, "[XXXXX10]": 42304, "[XXXXX358]": 42652, "[XXXXX109]": 42403, "[XXXXX407]": 42701, "[XXXXX296]": 42590, "[XXXXX246]": 42540, "[XXXXX105]": 42399, "[XXXXX609]": 42903, "[XXXXX228]": 42522, "[XXXXX148]": 42442, "[XXXXX600]": 42894, "[XXXXX110]": 42404, "[XXXXX692]": 42986, "[XXXXX443]": 42737, "[XXXXX262]": 42556, "[XXXXX625]": 42919, "[XXXXX258]": 42552, "<pad>": 42292, "[XXXXX436]": 42730, "[XXXXX199]": 42493, "[XXXXX315]": 42609, "[XXXXX570]": 42864, "[XXXXX678]": 42972, "[XXXXX466]": 42760, "[XXXXX560]": 42854, "[XXXXX650]": 42944, "[XXXXX167]": 42461, "[XXXXX213]": 42507, "[XXXXX16]": 42310, "[XXXXX78]": 42372, "[XXXXX121]": 42415, "[XXXXX342]": 42636, "[XXXXX79]": 42373, "[XXXXX318]": 42612, "[XXXXX577]": 42871, "[XXXXX346]": 42640, "[XXXXX266]": 42560, "[XXXXX65]": 42359, "[XXXXX61]": 42355, "[XXXXX627]": 42921, "[XXXXX567]": 42861, "[XXXXX20]": 42314, "[XXXXX277]": 42571, "[XXXXX232]": 42526, "[XXXXX116]": 42410, "[XXXXX361]": 42655, "[XXXXX581]": 42875, "[XXXXX41]": 42335, "[XXXXX279]": 42573, "[XXXXX143]": 42437, "[XXXXX549]": 42843, "[XXXXX623]": 42917, "[XXXXX465]": 42759, "[XXXXX225]": 42519, "[XXXXX368]": 42662, "[XXXXX96]": 42390, "[XXXXX98]": 42392, "[XXXXX37]": 42331, "[XXXXX442]": 42736, "[XXXXX259]": 42553, "[XXXXX492]": 42786, "[XXXXX647]": 42941, "[XXXXX411]": 42705, "[XXXXX257]": 42551, "[XXXXX350]": 42644, "[XXXXX654]": 42948, "[XXXXX523]": 42817, "[XXXXX639]": 42933, "[XXXXX239]": 42533, "[XXXXX561]": 42855, "[XXXXX54]": 42348, "[XXXXX4]": 42298, "[XXXXX520]": 42814, "[XXXXX319]": 42613, "[XXXXX303]": 42597, "[XXXXX363]": 42657, "[XXXXX572]": 42866, "[XXXXX185]": 42479, "[XXXXX322]": 42616, "[XXXXX526]": 42820, "[XXXXX18]": 42312, "[XXXXX617]": 42911, "[XXXXX651]": 42945, "[XXXXX23]": 42317, "[XXXXX31]": 42325, "[XXXXX231]": 42525, "[XXXXX459]": 42753, "[XXXXX308]": 42602, "[XXXXX85]": 42379, "[XXXXX530]": 42824, "[XXXXX390]": 42684, "[XXXXX282]": 42576, "[XXXXX307]": 42601, "[XXXXX375]": 42669, "[XXXXX688]": 42982, "[XXXXX365]": 42659, "[XXXXX546]": 42840, "[XXXXX314]": 42608, "[XXXXX274]": 42568, "[XXXXX473]": 42767, "[XXXXX333]": 42627, "[XXXXX114]": 42408, "[XXXXX60]": 42354, "[XXXXX551]": 42845, "[XXXXX544]": 42838, "[XXXXX24]": 42318, "[XXXXX108]": 42402, "[XXXXX453]": 42747, "[XXXXX404]": 42698, "[XXXXX111]": 42405, "[XXXXX289]": 42583, "[XXXXX435]": 42729, "[XXXXX115]": 42409, "[XXXXX611]": 42905, "[XXXXX543]": 42837, "[XXXXX372]": 42666, "[XXXXX667]": 42961, "[XXXXX584]": 42878, "[XXXXX124]": 42418, "[XXXXX505]": 42799, "[XXXXX87]": 42381, "[XXXXX519]": 42813, "[XXXXX491]": 42785, "[XXXXX475]": 42769, "[XXXXX86]": 42380, "[XXXXX43]": 42337, "[XXXXX312]": 42606, "[XXXXX255]": 42549, "[XXXXX12]": 42306, "[XXXXX424]": 42718, "[XXXXX433]": 42727, "[XXXXX489]": 42783, "[XXXXX515]": 42809, "[XXXXX270]": 42564, "[XXXXX462]": 42756, "[XXXXX388]": 42682, "[XXXXX80]": 42374, "[XXXXX641]": 42935, "[XXXXX159]": 42453, "[XXXXX94]": 42388, "[XXXXX141]": 42435, "[XXXXX73]": 42367, "[XXXXX200]": 42494, "[XXXXX535]": 42829, "[XXXXX90]": 42384, "[XXXXX252]": 42546, "[XXXXX698]": 42992, "[XXXXX638]": 42932, "[XXXXX95]": 42389, "[XXXXX500]": 42794, "[XXXXX679]": 42973, "[XXXXX485]": 42779, "[XXXXX488]": 42782, "[XXXXX281]": 42575, "[XXXXX93]": 42387, "[XXXXX604]": 42898, "[XXXXX71]": 42365, "[XXXXX120]": 42414, "[XXXXX151]": 42445, "[XXXXX38]": 42332, "[XXXXX445]": 42739, "[XXXXX222]": 42516, "[XXXXX712]": 43006, "[XXXXX104]": 42398, "[XXXXX455]": 42749, "[XXXXX242]": 42536, "[XXXXX5]": 42299, "[XXXXX230]": 42524, "[XXXXX653]": 42947, "[XXXXX260]": 42554, "[XXXXX405]": 42699, "[XXXXX214]": 42508, "[XXXXX298]": 42592, "[XXXXX382]": 42676, "[XXXXX508]": 42802, "[XXXXX662]": 42956, "[XXXXX438]": 42732, "[XXXXX190]": 42484, "[XXXXX440]": 42734, "[XXXXX599]": 42893, "[XXXXX337]": 42631, "[XXXXX648]": 42942, "[XXXXX253]": 42547, "[XXXXX619]": 42913, "[XXXXX381]": 42675, "[XXXXX17]": 42311, "[XXXXX152]": 42446, "[XXXXX100]": 42394, "[XXXXX40]": 42334, "[XXXXX29]": 42323, "[XXXXX45]": 42339, "[XXXXX568]": 42862, "[XXXXX518]": 42812, "[XXXXX129]": 42423, "[XXXXX474]": 42768, "[XXXXX655]": 42949, "[XXXXX160]": 42454, "[XXXXX383]": 42677, "[XXXXX385]": 42679, "[XXXXX285]": 42579, "[XXXXX198]": 42492, "[XXXXX507]": 42801, "[XXXXX613]": 42907, "[XXXXX675]": 42969, "[XXXXX302]": 42596, "[XXXXX642]": 42936, "[XXXXX103]": 42397, "[XXXXX444]": 42738, "[XXXXX573]": 42867, "[XXXXX467]": 42761, "[XXXXX295]": 42589, "[XXXXX202]": 42496, "[XXXXX173]": 42467, "[XXXXX412]": 42706, "[XXXXX369]": 42663, "[XXXXX217]": 42511, "[XXXXX132]": 42426, "[XXXXX169]": 42463, "[XXXXX497]": 42791, "[XXXXX673]": 42967, "[XXXXX379]": 42673, "[XXXXX541]": 42835, "[XXXXX97]": 42391, "[XXXXX236]": 42530, "[XXXXX636]": 42930, "[XXXXX659]": 42953, "[XXXXX704]": 42998, "[XXXXX165]": 42459, "[XXXXX409]": 42703, "[XXXXX331]": 42625, "[XXXXX707]": 43001, "[XXXXX463]": 42757, "[XXXXX588]": 42882, "[XXXXX127]": 42421, "[XXXXX664]": 42958, "[XXXXX499]": 42793, "[XXXXX512]": 42806, "[XXXXX421]": 42715, "[XXXXX84]": 42378, "[XXXXX532]": 42826, "[XXXXX163]": 42457}