goldfish-models commited on
Commit
4bdfc0f
1 Parent(s): 10524d7

Upload srn_latn_full tokenizer.

Browse files
added_tokens.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"[XXXXX967]": 28403, "[XXXXX1172]": 28608, "[XXXXX288]": 27724, "[XXXXX1141]": 28577, "[XXXXX1002]": 28438, "[XXXXX581]": 28017, "[XXXXX904]": 28340, "[XXXXX240]": 27676, "[XXXXX1189]": 28625, "[XXXXX846]": 28282, "[XXXXX337]": 27773, "[XXXXX545]": 27981, "[XXXXX296]": 27732, "[XXXXX1150]": 28586, "[XXXXX86]": 27522, "[XXXXX1112]": 28548, "[XXXXX412]": 27848, "[XXXXX608]": 28044, "[XXXXX992]": 28428, "[XXXXX475]": 27911, "[XXXXX74]": 27510, "[XXXXX115]": 27551, "[XXXXX482]": 27918, "[XXXXX1055]": 28491, "[XXXXX925]": 28361, "[XXXXX1132]": 28568, "[XXXXX880]": 28316, "[XXXXX1017]": 28453, "[XXXXX319]": 27755, "[XXXXX64]": 27500, "[XXXXX797]": 28233, "[XXXXX1093]": 28529, "[XXXXX1011]": 28447, "[XXXXX620]": 28056, "[XXXXX936]": 28372, "[XXXXX116]": 27552, "[XXXXX1154]": 28590, "[XXXXX397]": 27833, "[XXXXX1005]": 28441, "[XXXXX600]": 28036, "[XXXXX317]": 27753, "[XXXXX69]": 27505, "[XXXXX1001]": 28437, "[XXXXX674]": 28110, "[XXXXX77]": 27513, "[XXXXX923]": 28359, "[XXXXX761]": 28197, "[XXXXX978]": 28414, "[XXXXX593]": 28029, "[XXXXX964]": 28400, "[XXXXX490]": 27926, "[XXXXX1181]": 28617, "[XXXXX1032]": 28468, "[XXXXX1048]": 28484, "[XXXXX442]": 27878, "[XXXXX259]": 27695, "[XXXXX187]": 27623, "[XXXXX774]": 28210, "[XXXXX820]": 28256, "[XXXXX1007]": 28443, "[XXXXX524]": 27960, "[XXXXX742]": 28178, "[XXXXX178]": 27614, "[XXXXX291]": 27727, "[XXXXX30]": 27466, "[XXXXX677]": 28113, "[XXXXX708]": 28144, "[XXXXX1088]": 28524, "[XXXXX733]": 28169, "[XXXXX1075]": 28511, "[XXXXX1099]": 28535, "[XXXXX327]": 27763, "[XXXXX365]": 27801, "[XXXXX1122]": 28558, "[XXXXX1108]": 28544, "[XXXXX244]": 27680, "[XXXXX0]": 27436, "[XXXXX591]": 28027, "[XXXXX1174]": 28610, "[XXXXX54]": 27490, "[XXXXX159]": 27595, "[XXXXX1203]": 28639, "[XXXXX1229]": 28665, "[XXXXX948]": 28384, "[XXXXX956]": 28392, "[XXXXX783]": 28219, "[XXXXX907]": 28343, "[XXXXX132]": 27568, "[XXXXX402]": 27838, "[XXXXX734]": 28170, "[XXXXX141]": 27577, "[XXXXX505]": 27941, "[XXXXX1079]": 28515, "[XXXXX976]": 28412, "[XXXXX881]": 28317, "[XXXXX171]": 27607, "[XXXXX269]": 27705, "[XXXXX558]": 27994, "[XXXXX605]": 28041, "[XXXXX120]": 27556, "[XXXXX1047]": 28483, "[XXXXX1116]": 28552, "[XXXXX757]": 28193, "[XXXXX812]": 28248, "[XXXXX152]": 27588, "[XXXXX1022]": 28458, "[XXXXX241]": 27677, "[XXXXX447]": 27883, "[XXXXX11]": 27447, "[XXXXX258]": 27694, "[XXXXX456]": 27892, "[XXXXX908]": 28344, "[XXXXX1155]": 28591, "[XXXXX811]": 28247, "[XXXXX901]": 28337, "[XXXXX1139]": 28575, "[XXXXX737]": 28173, "[XXXXX903]": 28339, "[XXXXX336]": 27772, "[XXXXX448]": 27884, "[XXXXX670]": 28106, "[XXXXX247]": 27683, "[XXXXX931]": 28367, "[XXXXX107]": 27543, "[XXXXX1009]": 28445, "[XXXXX52]": 27488, "[XXXXX229]": 27665, "[XXXXX707]": 28143, "[XXXXX1071]": 28507, "[XXXXX882]": 28318, "[XXXXX577]": 28013, "[XXXXX798]": 28234, "[XXXXX684]": 28120, "[XXXXX1008]": 28444, "[XXXXX1040]": 28476, "[XXXXX859]": 28295, "[XXXXX42]": 27478, "[XXXXX1084]": 28520, "[XXXXX873]": 28309, "[XXXXX267]": 27703, "[XXXXX786]": 28222, "[XXXXX119]": 27555, "[XXXXX272]": 27708, "[XXXXX487]": 27923, "[XXXXX224]": 27660, "[XXXXX355]": 27791, "[XXXXX434]": 27870, "[XXXXX900]": 28336, "[XXXXX512]": 27948, "[XXXXX969]": 28405, "[XXXXX668]": 28104, "[XXXXX768]": 28204, "[XXXXX112]": 27548, "[XXXXX367]": 27803, "[XXXXX230]": 27666, "[XXXXX1068]": 28504, "[XXXXX1066]": 28502, "[XXXXX122]": 27558, "[XXXXX619]": 28055, "[XXXXX1058]": 28494, "[XXXXX1162]": 28598, "[XXXXX960]": 28396, "[XXXXX158]": 27594, "[XXXXX869]": 28305, "[XXXXX504]": 27940, "[XXXXX435]": 27871, "[XXXXX889]": 28325, "[XXXXX935]": 28371, "[XXXXX318]": 27754, "[XXXXX173]": 27609, "[XXXXX803]": 28239, "[XXXXX252]": 27688, "[XXXXX961]": 28397, "[XXXXX527]": 27963, "[XXXXX1081]": 28517, "[XXXXX756]": 28192, "[XXXXX533]": 27969, "[XXXXX617]": 28053, "[XXXXX519]": 27955, "[XXXXX541]": 27977, "[XXXXX949]": 28385, "[XXXXX452]": 27888, "[XXXXX603]": 28039, "[XXXXX831]": 28267, "[XXXXX300]": 27736, "[XXXXX926]": 28362, "[XXXXX257]": 27693, "[XXXXX245]": 27681, "[XXXXX993]": 28429, "[XXXXX825]": 28261, "[XXXXX658]": 28094, "[XXXXX566]": 28002, "[XXXXX387]": 27823, "[XXXXX255]": 27691, "[XXXXX19]": 27455, "[XXXXX902]": 28338, "[XXXXX535]": 27971, "[XXXXX457]": 27893, "[XXXXX1006]": 28442, "[XXXXX1126]": 28562, "[XXXXX822]": 28258, "[XXXXX865]": 28301, "[XXXXX91]": 27527, "[XXXXX762]": 28198, "[XXXXX842]": 28278, "[XXXXX419]": 27855, "[XXXXX648]": 28084, "[XXXXX832]": 28268, "[XXXXX1134]": 28570, "[XXXXX1127]": 28563, "[XXXXX744]": 28180, "[XXXXX962]": 28398, "[XXXXX262]": 27698, "[XXXXX389]": 27825, "[XXXXX1187]": 28623, "[XXXXX1202]": 28638, "[XXXXX530]": 27966, "[XXXXX521]": 27957, "[XXXXX947]": 28383, "[XXXXX726]": 28162, "[XXXXX968]": 28404, "[XXXXX1030]": 28466, "[XXXXX679]": 28115, "[XXXXX1119]": 28555, "[XXXXX1082]": 28518, "[XXXXX823]": 28259, "[XXXXX938]": 28374, "[XXXXX384]": 27820, "[XXXXX1057]": 28493, "[XXXXX622]": 28058, "[XXXXX223]": 27659, "[XXXXX814]": 28250, "[XXXXX3]": 27439, "[XXXXX913]": 28349, "[XXXXX884]": 28320, "[XXXXX374]": 27810, "[XXXXX283]": 27719, "[XXXXX789]": 28225, "[XXXXX211]": 27647, "[XXXXX872]": 28308, "[XXXXX777]": 28213, "[XXXXX87]": 27523, "[XXXXX1142]": 28578, "[XXXXX1219]": 28655, "[XXXXX916]": 28352, "[XXXXX5]": 27441, "[XXXXX237]": 27673, "[XXXXX1169]": 28605, "[XXXXX1235]": 28671, "[XXXXX1182]": 28618, "[XXXXX563]": 27999, "[XXXXX715]": 28151, "[XXXXX1227]": 28663, "[XXXXX1067]": 28503, "[XXXXX1121]": 28557, "[XXXXX1232]": 28668, "[XXXXX344]": 27780, "[XXXXX356]": 27792, "[XXXXX1046]": 28482, "[XXXXX463]": 27899, "[XXXXX590]": 28026, "[XXXXX1128]": 28564, "[XXXXX1222]": 28658, "[XXXXX186]": 27622, "[XXXXX154]": 27590, "[XXXXX532]": 27968, "[XXXXX156]": 27592, "[XXXXX1015]": 28451, "[XXXXX1125]": 28561, "[XXXXX791]": 28227, "[XXXXX775]": 28211, "[XXXXX453]": 27889, "[XXXXX151]": 27587, "[XXXXX910]": 28346, "[XXXXX280]": 27716, "[XXXXX728]": 28164, "[XXXXX1024]": 28460, "[XXXXX833]": 28269, "[XXXXX933]": 28369, "[XXXXX623]": 28059, "[XXXXX1184]": 28620, "[XXXXX287]": 27723, "[XXXXX213]": 27649, "[XXXXX108]": 27544, "[XXXXX574]": 28010, "[XXXXX893]": 28329, "[XXXXX286]": 27722, "[XXXXX218]": 27654, "[XXXXX185]": 27621, "[XXXXX796]": 28232, "[CLS]": 27432, "[XXXXX699]": 28135, "[XXXXX525]": 27961, "[XXXXX375]": 27811, "[XXXXX222]": 27658, "[XXXXX314]": 27750, "[XXXXX800]": 28236, "[XXXXX909]": 28345, "[XXXXX944]": 28380, "[XXXXX1179]": 28615, "[XXXXX929]": 28365, "[XXXXX883]": 28319, "[XXXXX446]": 27882, "[XXXXX261]": 27697, "[XXXXX544]": 27980, "[XXXXX1226]": 28662, "[XXXXX813]": 28249, "[XXXXX46]": 27482, "[XXXXX433]": 27869, "[XXXXX749]": 28185, "[XXXXX364]": 27800, "[XXXXX1056]": 28492, "[XXXXX646]": 28082, "[XXXXX934]": 28370, "[XXXXX1003]": 28439, "[XXXXX395]": 27831, "[XXXXX444]": 27880, "[XXXXX88]": 27524, "[XXXXX989]": 28425, "[XXXXX138]": 27574, "[XXXXX876]": 28312, "[XXXXX137]": 27573, "[XXXXX194]": 27630, "[XXXXX164]": 27600, "[XXXXX1156]": 28592, "[XXXXX687]": 28123, "[XXXXX1098]": 28534, "[XXXXX542]": 27978, "[XXXXX1010]": 28446, "[XXXXX1110]": 28546, "[XXXXX22]": 27458, "[XXXXX851]": 28287, "[XXXXX163]": 27599, "[XXXXX1131]": 28567, "[XXXXX693]": 28129, "[XXXXX1102]": 28538, "[XXXXX582]": 28018, "[XXXXX407]": 27843, "[XXXXX236]": 27672, "[XXXXX301]": 27737, "[XXXXX705]": 28141, "[XXXXX481]": 27917, "[XXXXX695]": 28131, "[XXXXX13]": 27449, "[XXXXX75]": 27511, "[XXXXX780]": 28216, "[XXXXX1091]": 28527, "[XXXXX930]": 28366, "[XXXXX1178]": 28614, "[XXXXX970]": 28406, "[XXXXX473]": 27909, "[XXXXX837]": 28273, "[XXXXX1044]": 28480, "[XXXXX503]": 27939, "[XXXXX35]": 27471, "[XXXXX373]": 27809, "[XXXXX1065]": 28501, "[XXXXX1085]": 28521, "[XXXXX292]": 27728, "[XXXXX468]": 27904, "[XXXXX1228]": 28664, "[XXXXX755]": 28191, "[XXXXX781]": 28217, "[XXXXX72]": 27508, "[XXXXX205]": 27641, "[XXXXX256]": 27692, "[XXXXX951]": 28387, "[XXXXX598]": 28034, "[XXXXX514]": 27950, "[XXXXX539]": 27975, "[XXXXX1049]": 28485, "[XXXXX202]": 27638, "[XXXXX131]": 27567, "[XXXXX655]": 28091, "[XXXXX785]": 28221, "[XXXXX144]": 27580, "[XXXXX1223]": 28659, "[XXXXX228]": 27664, "[XXXXX946]": 28382, "[XXXXX1171]": 28607, "[XXXXX31]": 27467, "[XXXXX200]": 27636, "[XXXXX61]": 27497, "[XXXXX1072]": 28508, "[XXXXX458]": 27894, "[XXXXX63]": 27499, "[XXXXX243]": 27679, "[XXXXX316]": 27752, "[XXXXX165]": 27601, "[XXXXX216]": 27652, "[XXXXX250]": 27686, "[XXXXX652]": 28088, "[XXXXX188]": 27624, "[XXXXX899]": 28335, "[XXXXX994]": 28430, "[XXXXX1183]": 28619, "[XXXXX410]": 27846, "[XXXXX313]": 27749, "[XXXXX1053]": 28489, "[XXXXX888]": 28324, "[XXXXX690]": 28126, "[XXXXX905]": 28341, "[XXXXX226]": 27662, "[XXXXX303]": 27739, "[XXXXX348]": 27784, "[XXXXX849]": 28285, "[XXXXX717]": 28153, "[XXXXX647]": 28083, "[XXXXX716]": 28152, "[XXXXX445]": 27881, "[XXXXX584]": 28020, "[XXXXX1103]": 28539, "[XXXXX499]": 27935, "[XXXXX436]": 27872, "[XXXXX586]": 28022, "[XXXXX980]": 28416, "[XXXXX1039]": 28475, "[XXXXX592]": 28028, "[XXXXX39]": 27475, "[XXXXX704]": 28140, "[XXXXX443]": 27879, "[XXXXX636]": 28072, "[XXXXX1140]": 28576, "[XXXXX248]": 27684, "[XXXXX51]": 27487, "[XXXXX342]": 27778, "[XXXXX360]": 27796, "[XXXXX294]": 27730, "[XXXXX306]": 27742, "[XXXXX26]": 27462, "[XXXXX706]": 28142, "[XXXXX363]": 27799, "[XXXXX209]": 27645, "[XXXXX289]": 27725, "[XXXXX1218]": 28654, "[XXXXX460]": 27896, "[XXXXX540]": 27976, "[XXXXX875]": 28311, "[XXXXX381]": 27817, "[XXXXX645]": 28081, "[XXXXX579]": 28015, "[XXXXX710]": 28146, "[XXXXX369]": 27805, "[XXXXX1]": 27437, "[XXXXX372]": 27808, "[XXXXX838]": 28274, "[XXXXX190]": 27626, "[XXXXX308]": 27744, "[XXXXX1173]": 28609, "[XXXXX1193]": 28629, "[XXXXX871]": 28307, "<pad>": 27434, "[XXXXX815]": 28251, "[XXXXX422]": 27858, "[XXXXX810]": 28246, "[XXXXX96]": 27532, "[XXXXX681]": 28117, "[XXXXX867]": 28303, "[XXXXX945]": 28381, "[XXXXX537]": 27973, "[XXXXX1204]": 28640, "[XXXXX1051]": 28487, "[XXXXX937]": 28373, "[XXXXX1070]": 28506, "[XXXXX917]": 28353, "[XXXXX297]": 27733, "[XXXXX638]": 28074, "[XXXXX18]": 27454, "[XXXXX996]": 28432, "[XXXXX624]": 28060, "[XXXXX1225]": 28661, "[XXXXX731]": 28167, "[XXXXX478]": 27914, "[XXXXX549]": 27985, "[XXXXX440]": 27876, "[XXXXX1004]": 28440, "[XXXXX214]": 27650, "[XXXXX66]": 27502, "[XXXXX98]": 27534, "[XXXXX110]": 27546, "[XXXXX147]": 27583, "[XXXXX41]": 27477, "[XXXXX701]": 28137, "[XXXXX1087]": 28523, "[XXXXX1206]": 28642, "[XXXXX1147]": 28583, "[XXXXX555]": 27991, "[XXXXX385]": 27821, "[XXXXX712]": 28148, "[XXXXX134]": 27570, "[XXXXX1020]": 28456, "[XXXXX1180]": 28616, "[XXXXX351]": 27787, "[XXXXX432]": 27868, "[XXXXX991]": 28427, "[XXXXX852]": 28288, "[XXXXX391]": 27827, "[XXXXX575]": 28011, "[XXXXX769]": 28205, "[XXXXX1037]": 28473, "[XXXXX776]": 28212, "[XXXXX493]": 27929, "[XXXXX747]": 28183, "[XXXXX887]": 28323, "[XXXXX246]": 27682, "[XXXXX1213]": 28649, "[XXXXX341]": 27777, "[XXXXX49]": 27485, "[XXXXX1123]": 28559, "[XXXXX27]": 27463, "[XXXXX350]": 27786, "[XXXXX454]": 27890, "[XXXXX920]": 28356, "[XXXXX362]": 27798, "[XXXXX782]": 28218, "[XXXXX191]": 27627, "[XXXXX390]": 27826, "[XXXXX225]": 27661, "[XXXXX78]": 27514, "[XXXXX1052]": 28488, "[XXXXX689]": 28125, "[XXXXX808]": 28244, "[XXXXX656]": 28092, "[XXXXX1163]": 28599, "[XXXXX725]": 28161, "[XXXXX597]": 28033, "[XXXXX678]": 28114, "[XXXXX741]": 28177, "[XXXXX799]": 28235, "[XXXXX204]": 27640, "[XXXXX1224]": 28660, "[XXXXX239]": 27675, "[XXXXX174]": 27610, "[XXXXX721]": 28157, "[XXXXX719]": 28155, "[XXXXX609]": 28045, "[XXXXX516]": 27952, "[XXXXX143]": 27579, "[XXXXX28]": 27464, "[XXXXX827]": 28263, "[XXXXX954]": 28390, "[XXXXX383]": 27819, "[XXXXX497]": 27933, "[XXXXX393]": 27829, "[XXXXX616]": 28052, "[XXXXX809]": 28245, "[XXXXX682]": 28118, "[XXXXX104]": 27540, "[XXXXX254]": 27690, "[XXXXX939]": 28375, "[XXXXX411]": 27847, "[XXXXX1038]": 28474, "[XXXXX711]": 28147, "[XXXXX1201]": 28637, "[XXXXX4]": 27440, "[XXXXX1230]": 28666, "[XXXXX1233]": 28669, "[XXXXX538]": 27974, "[XXXXX713]": 28149, "[SEP]": 27433, "[XXXXX404]": 27840, "[XXXXX36]": 27472, "[XXXXX1083]": 28519, "[XXXXX1198]": 28634, "[XXXXX739]": 28175, "[XXXXX126]": 27562, "[XXXXX81]": 27517, "[XXXXX428]": 27864, "[XXXXX146]": 27582, "[XXXXX24]": 27460, "[XXXXX659]": 28095, "[XXXXX451]": 27887, "[XXXXX860]": 28296, "[XXXXX449]": 27885, "[XXXXX943]": 28379, "[XXXXX339]": 27775, "[XXXXX71]": 27507, "[XXXXX1160]": 28596, "[XXXXX760]": 28196, "[XXXXX661]": 28097, "[XXXXX853]": 28289, "[XXXXX644]": 28080, "[XXXXX621]": 28057, "[XXXXX414]": 27850, "[XXXXX150]": 27586, "[XXXXX1136]": 28572, "[XXXXX462]": 27898, "[XXXXX338]": 27774, "[XXXXX325]": 27761, "[XXXXX8]": 27444, "[XXXXX1107]": 28543, "[XXXXX290]": 27726, "[XXXXX1208]": 28644, "[XXXXX215]": 27651, "[XXXXX650]": 28086, "[XXXXX1146]": 28582, "[XXXXX57]": 27493, "[XXXXX714]": 28150, "[XXXXX618]": 28054, "[XXXXX751]": 28187, "[XXXXX305]": 27741, "[XXXXX263]": 27699, "[XXXXX1231]": 28667, "[XXXXX359]": 27795, "[XXXXX1094]": 28530, "[XXXXX1025]": 28461, "[XXXXX639]": 28075, "[XXXXX520]": 27956, "[XXXXX890]": 28326, "[XXXXX417]": 27853, "[XXXXX765]": 28201, "[XXXXX564]": 28000, "[XXXXX779]": 28215, "[XXXXX466]": 27902, "[XXXXX856]": 28292, "[XXXXX136]": 27572, "[XXXXX192]": 27628, "[XXXXX56]": 27492, "[XXXXX129]": 27565, "[XXXXX489]": 27925, "[XXXXX101]": 27537, "[XXXXX818]": 28254, "[XXXXX529]": 27965, "[XXXXX1043]": 28479, "[XXXXX979]": 28415, "[XXXXX641]": 28077, "[XXXXX92]": 27528, "[XXXXX952]": 28388, "[XXXXX1076]": 28512, "[XXXXX201]": 27637, "[XXXXX322]": 27758, "[XXXXX465]": 27901, "[XXXXX464]": 27900, "[XXXXX675]": 28111, "[XXXXX496]": 27932, "[XXXXX324]": 27760, "[XXXXX517]": 27953, "[XXXXX130]": 27566, "[XXXXX686]": 28122, "[XXXXX168]": 27604, "[XXXXX7]": 27443, "[XXXXX1078]": 28514, "[XXXXX754]": 28190, "[XXXXX1092]": 28528, "[XXXXX919]": 28355, "[XXXXX528]": 27964, "[XXXXX1086]": 28522, "[XXXXX279]": 27715, "[XXXXX1059]": 28495, "[XXXXX472]": 27908, "[XXXXX135]": 27571, "[XXXXX331]": 27767, "[XXXXX208]": 27644, "[XXXXX870]": 28306, "[XXXXX942]": 28378, "[XXXXX732]": 28168, "[XXXXX984]": 28420, "[XXXXX346]": 27782, "[XXXXX1176]": 28612, "[XXXXX196]": 27632, "[XXXXX703]": 28139, "[XXXXX479]": 27915, "[XXXXX184]": 27620, "[XXXXX426]": 27862, "[XXXXX148]": 27584, "[XXXXX1035]": 28471, "[XXXXX58]": 27494, "[XXXXX767]": 28203, "[XXXXX276]": 27712, "[XXXXX65]": 27501, "[XXXXX510]": 27946, "[XXXXX998]": 28434, "[XXXXX34]": 27470, "[XXXXX759]": 28195, "[XXXXX1133]": 28569, "[XXXXX323]": 27759, "[XXXXX676]": 28112, "[XXXXX170]": 27606, "[XXXXX625]": 28061, "[XXXXX1026]": 28462, "[XXXXX629]": 28065, "[XXXXX40]": 27476, "[XXXXX408]": 27844, "[XXXXX38]": 27474, "[XXXXX801]": 28237, "[XXXXX1209]": 28645, "[XXXXX349]": 27785, "[XXXXX1113]": 28549, "[XXXXX561]": 27997, "[XXXXX986]": 28422, "[XXXXX197]": 27633, "[XXXXX47]": 27483, "[XXXXX1194]": 28630, "[XXXXX843]": 28279, "[XXXXX1143]": 28579, "[XXXXX643]": 28079, "[XXXXX76]": 27512, "[XXXXX310]": 27746, "[XXXXX745]": 28181, "[XXXXX1089]": 28525, "[XXXXX1069]": 28505, "[XXXXX461]": 27897, "[XXXXX492]": 27928, "[XXXXX1031]": 28467, "[XXXXX1190]": 28626, "[XXXXX260]": 27696, "[MASK]": 27435, "[XXXXX315]": 27751, "[XXXXX578]": 28014, "[XXXXX491]": 27927, "[XXXXX93]": 27529, "[XXXXX770]": 28206, "[XXXXX1018]": 28454, "[XXXXX498]": 27934, "[XXXXX892]": 28328, "[XXXXX866]": 28302, "[XXXXX212]": 27648, "[XXXXX848]": 28284, "[XXXXX594]": 28030, "[XXXXX806]": 28242, "[XXXXX633]": 28069, "[XXXXX839]": 28275, "[XXXXX1115]": 28551, "[XXXXX548]": 27984, "[XXXXX988]": 28424, "[XXXXX175]": 27611, "[XXXXX450]": 27886, "[XXXXX612]": 28048, "[XXXXX427]": 27863, "[XXXXX599]": 28035, "[XXXXX522]": 27958, "[XXXXX534]": 27970, "[XXXXX21]": 27457, "[XXXXX874]": 28310, "[XXXXX177]": 27613, "[XXXXX379]": 27815, "[XXXXX242]": 27678, "[XXXXX1021]": 28457, "[XXXXX634]": 28070, "[XXXXX1164]": 28600, "[XXXXX284]": 27720, "[XXXXX206]": 27642, "[XXXXX1023]": 28459, "[XXXXX778]": 28214, "[XXXXX855]": 28291, "[XXXXX125]": 27561, "[XXXXX485]": 27921, "[XXXXX413]": 27849, "[XXXXX409]": 27845, "[XXXXX557]": 27993, "[XXXXX1109]": 28545, "[XXXXX1114]": 28550, "[XXXXX1165]": 28601, "[XXXXX666]": 28102, "[XXXXX118]": 27554, "[XXXXX788]": 28224, "[XXXXX1095]": 28531, "[XXXXX1200]": 28636, "[XXXXX573]": 28009, "[XXXXX854]": 28290, "[XXXXX398]": 27834, "[XXXXX352]": 27788, "[XXXXX841]": 28277, "[XXXXX642]": 28078, "[XXXXX1054]": 28490, "[XXXXX698]": 28134, "[XXXXX1214]": 28650, "[XXXXX1027]": 28463, "[XXXXX1211]": 28647, "[XXXXX752]": 28188, "[XXXXX587]": 28023, "[XXXXX1041]": 28477, "[XXXXX354]": 27790, "[XXXXX958]": 28394, "[XXXXX176]": 27612, "[XXXXX1207]": 28643, "[XXXXX1073]": 28509, "[XXXXX180]": 27616, "[XXXXX423]": 27859, "[XXXXX488]": 27924, "[XXXXX977]": 28413, "[XXXXX793]": 28229, "[XXXXX585]": 28021, "[XXXXX914]": 28350, "[XXXXX671]": 28107, "[XXXXX62]": 27498, "[XXXXX615]": 28051, "[XXXXX16]": 27452, "[XXXXX357]": 27793, "[XXXXX829]": 28265, "[XXXXX139]": 27575, "[XXXXX828]": 28264, "[XXXXX29]": 27465, "[XXXXX569]": 28005, "[XXXXX117]": 27553, "[XXXXX894]": 28330, "[XXXXX45]": 27481, "[XXXXX439]": 27875, "[XXXXX1220]": 28656, "[XXXXX673]": 28109, "[XXXXX651]": 28087, "[XXXXX271]": 27707, "[XXXXX189]": 27625, "[XXXXX149]": 27585, "[XXXXX1158]": 28594, "[XXXXX83]": 27519, "[XXXXX1188]": 28624, "[XXXXX1042]": 28478, "[XXXXX507]": 27943, "[XXXXX911]": 28347, "[XXXXX915]": 28351, "[XXXXX312]": 27748, "[XXXXX565]": 28001, "[XXXXX438]": 27874, "[XXXXX1157]": 28593, "[XXXXX1036]": 28472, "[XXXXX368]": 27804, "[XXXXX807]": 28243, "[XXXXX161]": 27597, "[XXXXX552]": 27988, "[XXXXX1199]": 28635, "[XXXXX1019]": 28455, "[XXXXX353]": 27789, "[XXXXX963]": 28399, "[XXXXX1186]": 28622, "[XXXXX264]": 27700, "[XXXXX415]": 27851, "[XXXXX302]": 27738, "[XXXXX567]": 28003, "[XXXXX1153]": 28589, "[XXXXX321]": 27757, "[XXXXX723]": 28159, "[XXXXX637]": 28073, "[XXXXX2]": 27438, "[XXXXX366]": 27802, "[XXXXX199]": 27635, "[XXXXX953]": 28389, "[XXXXX583]": 28019, "[XXXXX6]": 27442, "[XXXXX688]": 28124, "[XXXXX700]": 28136, "[XXXXX376]": 27812, "[XXXXX311]": 27747, "[XXXXX610]": 28046, "[XXXXX736]": 28172, "[XXXXX80]": 27516, "[XXXXX792]": 28228, "[XXXXX740]": 28176, "[XXXXX89]": 27525, "[XXXXX526]": 27962, "[XXXXX12]": 27448, "[XXXXX897]": 28333, "[XXXXX1111]": 28547, "[XXXXX102]": 27538, "[XXXXX437]": 27873, "[XXXXX763]": 28199, "[XXXXX604]": 28040, "[XXXXX868]": 28304, "[XXXXX589]": 28025, "[XXXXX97]": 27533, "[XXXXX1129]": 28565, "[XXXXX330]": 27766, "[XXXXX1014]": 28450, "[XXXXX1148]": 28584, "[XXXXX836]": 28272, "[XXXXX23]": 27459, "[XXXXX824]": 28260, "[XXXXX669]": 28105, "[XXXXX1234]": 28670, "[XXXXX660]": 28096, "[XXXXX1061]": 28497, "[XXXXX1175]": 28611, "[XXXXX293]": 27729, "[XXXXX1210]": 28646, "[XXXXX401]": 27837, "[XXXXX219]": 27655, "[XXXXX766]": 28202, "[XXXXX123]": 27559, "[XXXXX1034]": 28470, "[XXXXX1028]": 28464, "[XXXXX142]": 27578, "[XXXXX845]": 28281, "[XXXXX418]": 27854, "[XXXXX425]": 27861, "[XXXXX928]": 28364, "[XXXXX626]": 28062, "[XXXXX329]": 27765, "[XXXXX394]": 27830, "[XXXXX1130]": 28566, "[XXXXX430]": 27866, "[XXXXX940]": 28376, "[XXXXX429]": 27865, "[XXXXX862]": 28298, "[XXXXX990]": 28426, "[XXXXX680]": 28116, "[XXXXX1161]": 28597, "[XXXXX1205]": 28641, "[XXXXX1168]": 28604, "[XXXXX483]": 27919, "[XXXXX743]": 28179, "[XXXXX231]": 27667, "[XXXXX33]": 27469, "[XXXXX804]": 28240, "[XXXXX44]": 27480, "[XXXXX576]": 28012, "[XXXXX295]": 27731, "[XXXXX59]": 27495, "[XXXXX377]": 27813, "[XXXXX898]": 28334, "[XXXXX275]": 27711, "[XXXXX399]": 27835, "[XXXXX79]": 27515, "[XXXXX37]": 27473, "[XXXXX1064]": 28500, "[XXXXX536]": 27972, "[XXXXX702]": 28138, "[XXXXX997]": 28433, "[XXXXX570]": 28006, "[XXXXX153]": 27589, "[XXXXX1215]": 28651, "[XXXXX1152]": 28588, "[XXXXX1191]": 28627, "[XXXXX48]": 27484, "[XXXXX662]": 28098, "[XXXXX982]": 28418, "[XXXXX469]": 27905, "[XXXXX653]": 28089, "[XXXXX392]": 27828, "[XXXXX957]": 28393, "[XXXXX157]": 27593, "[XXXXX416]": 27852, "[XXXXX340]": 27776, "[XXXXX562]": 27998, "[XXXXX207]": 27643, "[XXXXX345]": 27781, "[XXXXX484]": 27920, "[XXXXX955]": 28391, "[XXXXX282]": 27718, "[XXXXX441]": 27877, "[XXXXX382]": 27818, "[XXXXX1137]": 28573, "[XXXXX965]": 28401, "[XXXXX32]": 27468, "[XXXXX73]": 27509, "[XXXXX1063]": 28499, "[XXXXX1212]": 28648, "[XXXXX114]": 27550, "[XXXXX25]": 27461, "[XXXXX895]": 28331, "[XXXXX470]": 27906, "[XXXXX746]": 28182, "[XXXXX1029]": 28465, "[XXXXX1077]": 28513, "[XXXXX1013]": 28449, "[XXXXX113]": 27549, "[XXXXX238]": 27674, "[XXXXX480]": 27916, "[XXXXX1177]": 28613, "[XXXXX864]": 28300, "[XXXXX431]": 27867, "[XXXXX198]": 27634, "[XXXXX729]": 28165, "[XXXXX1166]": 28602, "[XXXXX922]": 28358, "[XXXXX309]": 27745, "[XXXXX370]": 27806, "[XXXXX1012]": 28448, "[XXXXX501]": 27937, "[XXXXX278]": 27714, "[XXXXX128]": 27564, "[XXXXX477]": 27913, "[XXXXX17]": 27453, "[XXXXX100]": 27536, "[XXXXX821]": 28257, "[XXXXX53]": 27489, "[XXXXX631]": 28067, "[XXXXX1149]": 28585, "[XXXXX167]": 27603, "[XXXXX607]": 28043, "[XXXXX1151]": 28587, "[XXXXX847]": 28283, "[XXXXX235]": 27671, "[XXXXX1167]": 28603, "[XXXXX103]": 27539, "[XXXXX758]": 28194, "[XXXXX974]": 28410, "[XXXXX912]": 28348, "[XXXXX941]": 28377, "[XXXXX819]": 28255, "[XXXXX326]": 27762, "[XXXXX794]": 28230, "[XXXXX857]": 28293, "[XXXXX654]": 28090, "[XXXXX543]": 27979, "[XXXXX494]": 27930, "[XXXXX983]": 28419, "[XXXXX1060]": 28496, "[XXXXX602]": 28038, "[XXXXX568]": 28004, "[XXXXX388]": 27824, "[XXXXX1124]": 28560, "[XXXXX495]": 27931, "[XXXXX333]": 27769, "[XXXXX790]": 28226, "[XXXXX877]": 28313, "[XXXXX234]": 27670, "[XXXXX614]": 28050, "[XXXXX281]": 27717, "[XXXXX1062]": 28498, "[XXXXX227]": 27663, "[XXXXX1138]": 28574, "[XXXXX878]": 28314, "[XXXXX511]": 27947, "[XXXXX1033]": 28469, "[XXXXX396]": 27832, "[XXXXX1192]": 28628, "[XXXXX1100]": 28536, "[XXXXX720]": 28156, "[XXXXX921]": 28357, "[XXXXX596]": 28032, "[XXXXX162]": 27598, "[XXXXX181]": 27617, "[XXXXX773]": 28209, "[XXXXX1097]": 28533, "[XXXXX973]": 28409, "[XXXXX420]": 27856, "[XXXXX471]": 27907, "[XXXXX109]": 27545, "[XXXXX332]": 27768, "[XXXXX795]": 28231, "[XXXXX500]": 27936, "[XXXXX335]": 27771, "[XXXXX1105]": 28541, "[XXXXX424]": 27860, "[XXXXX304]": 27740, "[XXXXX685]": 28121, "[XXXXX613]": 28049, "[XXXXX182]": 27618, "[XXXXX861]": 28297, "[XXXXX547]": 27983, "[XXXXX60]": 27496, "[XXXXX405]": 27841, "[XXXXX1159]": 28595, "[XXXXX817]": 28253, "[XXXXX124]": 27560, "[XXXXX1120]": 28556, "[XXXXX380]": 27816, "[XXXXX1045]": 28481, "[XXXXX421]": 27857, "[XXXXX772]": 28208, "[XXXXX1118]": 28554, "[XXXXX697]": 28133, "[XXXXX556]": 27992, "[XXXXX683]": 28119, "[XXXXX406]": 27842, "[XXXXX826]": 28262, "[XXXXX672]": 28108, "[XXXXX753]": 28189, "[XXXXX891]": 28327, "[XXXXX764]": 28200, "[XXXXX692]": 28128, "[XXXXX509]": 27945, "[XXXXX571]": 28007, "[XXXXX987]": 28423, "[XXXXX858]": 28294, "[XXXXX802]": 28238, "[XXXXX1090]": 28526, "[XXXXX195]": 27631, "[XXXXX1145]": 28581, "[XXXXX816]": 28252, "[XXXXX400]": 27836, "[XXXXX611]": 28047, "[XXXXX835]": 28271, "[XXXXX172]": 27608, "[XXXXX1216]": 28652, "[XXXXX550]": 27986, "[XXXXX724]": 28160, "[XXXXX328]": 27764, "[XXXXX179]": 27615, "[XXXXX467]": 27903, "[XXXXX85]": 27521, "[XXXXX111]": 27547, "[XXXXX307]": 27743, "[XXXXX513]": 27949, "[XXXXX9]": 27445, "[XXXXX580]": 28016, "[XXXXX105]": 27541, "[XXXXX285]": 27721, "[XXXXX630]": 28066, "[XXXXX1197]": 28633, "[XXXXX320]": 27756, "[XXXXX155]": 27591, "[XXXXX830]": 28266, "[XXXXX834]": 28270, "[XXXXX995]": 28431, "[XXXXX1196]": 28632, "[XXXXX251]": 27687, "[XXXXX455]": 27891, "[XXXXX840]": 28276, "[XXXXX730]": 28166, "[XXXXX127]": 27563, "[XXXXX343]": 27779, "[XXXXX476]": 27912, "[XXXXX657]": 28093, "[XXXXX748]": 28184, "[XXXXX277]": 27713, "[XXXXX1106]": 28542, "[XXXXX1221]": 28657, "[XXXXX99]": 27535, "[XXXXX667]": 28103, "[XXXXX727]": 28163, "[XXXXX220]": 27656, "[XXXXX850]": 28286, "[XXXXX270]": 27706, "[XXXXX1096]": 28532, "[XXXXX358]": 27794, "[XXXXX90]": 27526, "[XXXXX1217]": 28653, "[XXXXX927]": 28363, "[XXXXX981]": 28417, "[XXXXX361]": 27797, "[XXXXX160]": 27596, "[XXXXX68]": 27504, "[XXXXX1135]": 28571, "[XXXXX665]": 28101, "[XXXXX502]": 27938, "[XXXXX50]": 27486, "[XXXXX15]": 27451, "[XXXXX750]": 28186, "[XXXXX1101]": 28537, "[XXXXX133]": 27569, "[XXXXX334]": 27770, "[XXXXX718]": 28154, "[XXXXX1000]": 28436, "[XXXXX210]": 27646, "[XXXXX70]": 27506, "[XXXXX595]": 28031, "[XXXXX233]": 27669, "[XXXXX221]": 27657, "[XXXXX55]": 27491, "[XXXXX551]": 27987, "[XXXXX709]": 28145, "[XXXXX924]": 28360, "[XXXXX950]": 28386, "[XXXXX918]": 28354, "[XXXXX1050]": 28486, "[XXXXX649]": 28085, "[XXXXX508]": 27944, "[XXXXX232]": 27668, "[XXXXX518]": 27954, "[XXXXX140]": 27576, "[XXXXX14]": 27450, "[XXXXX273]": 27709, "[XXXXX403]": 27839, "[XXXXX67]": 27503, "[XXXXX663]": 28099, "[XXXXX121]": 27557, "[XXXXX531]": 27967, "[XXXXX1074]": 28510, "[XXXXX696]": 28132, "[XXXXX906]": 28342, "[XXXXX885]": 28321, "[XXXXX183]": 27619, "[XXXXX896]": 28332, "[XXXXX738]": 28174, "[XXXXX84]": 27520, "[XXXXX299]": 27735, "[XXXXX298]": 27734, "[XXXXX94]": 27530, "[XXXXX20]": 27456, "[XXXXX691]": 28127, "[XXXXX1080]": 28516, "[XXXXX474]": 27910, "[XXXXX560]": 27996, "[XXXXX628]": 28064, "[XXXXX515]": 27951, "[XXXXX879]": 28315, "[XXXXX371]": 27807, "[XXXXX664]": 28100, "[XXXXX193]": 27629, "[XXXXX735]": 28171, "[XXXXX784]": 28220, "[XXXXX217]": 27653, "[XXXXX169]": 27605, "[XXXXX601]": 28037, "[XXXXX1195]": 28631, "[XXXXX145]": 27581, "[XXXXX985]": 28421, "[XXXXX966]": 28402, "[XXXXX635]": 28071, "[XXXXX975]": 28411, "[XXXXX886]": 28322, "[XXXXX722]": 28158, "[XXXXX1144]": 28580, "[XXXXX274]": 27710, "[XXXXX378]": 27814, "[XXXXX1117]": 28553, "[XXXXX559]": 27995, "[XXXXX106]": 27542, "[XXXXX386]": 27822, "[XXXXX972]": 28408, "[XXXXX10]": 27446, "[XXXXX203]": 27639, "[XXXXX82]": 27518, "[XXXXX959]": 28395, "[XXXXX546]": 27982, "[XXXXX266]": 27702, "[XXXXX265]": 27701, "[XXXXX166]": 27602, "[XXXXX1016]": 28452, "[XXXXX268]": 27704, "[XXXXX347]": 27783, "[XXXXX249]": 27685, "[XXXXX805]": 28241, "[XXXXX627]": 28063, "[XXXXX1170]": 28606, "[XXXXX506]": 27942, "[XXXXX43]": 27479, "[XXXXX932]": 28368, "[XXXXX588]": 28024, "[XXXXX787]": 28223, "[XXXXX694]": 28130, "[XXXXX554]": 27990, "[XXXXX632]": 28068, "[XXXXX1104]": 28540, "[XXXXX459]": 27895, "[XXXXX606]": 28042, "[XXXXX486]": 27922, "[XXXXX971]": 28407, "[XXXXX863]": 28299, "[XXXXX553]": 27989, "[XXXXX523]": 27959, "[XXXXX771]": 28207, "[XXXXX95]": 27531, "[XXXXX572]": 28008, "[XXXXX999]": 28435, "[XXXXX844]": 28280, "[XXXXX1185]": 28621, "[XXXXX253]": 27689, "[XXXXX640]": 28076}
special_tokens_map.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "<unk>", "sep_token": "[SEP]", "pad_token": "<pad>", "cls_token": "[CLS]", "mask_token": {"content": "[MASK]", "single_word": false, "lstrip": true, "rstrip": false, "normalized": false}, "additional_special_tokens": ["[XXXXX0]", "[XXXXX1]", "[XXXXX2]", "[XXXXX3]", "[XXXXX4]", "[XXXXX5]", "[XXXXX6]", "[XXXXX7]", "[XXXXX8]", "[XXXXX9]", "[XXXXX10]", "[XXXXX11]", "[XXXXX12]", "[XXXXX13]", "[XXXXX14]", "[XXXXX15]", "[XXXXX16]", "[XXXXX17]", "[XXXXX18]", "[XXXXX19]", "[XXXXX20]", "[XXXXX21]", "[XXXXX22]", "[XXXXX23]", "[XXXXX24]", "[XXXXX25]", "[XXXXX26]", "[XXXXX27]", "[XXXXX28]", "[XXXXX29]", "[XXXXX30]", "[XXXXX31]", "[XXXXX32]", "[XXXXX33]", "[XXXXX34]", "[XXXXX35]", "[XXXXX36]", "[XXXXX37]", "[XXXXX38]", "[XXXXX39]", "[XXXXX40]", "[XXXXX41]", "[XXXXX42]", "[XXXXX43]", "[XXXXX44]", "[XXXXX45]", "[XXXXX46]", "[XXXXX47]", "[XXXXX48]", "[XXXXX49]", "[XXXXX50]", "[XXXXX51]", "[XXXXX52]", "[XXXXX53]", "[XXXXX54]", "[XXXXX55]", "[XXXXX56]", "[XXXXX57]", "[XXXXX58]", "[XXXXX59]", "[XXXXX60]", "[XXXXX61]", "[XXXXX62]", "[XXXXX63]", "[XXXXX64]", "[XXXXX65]", "[XXXXX66]", "[XXXXX67]", "[XXXXX68]", "[XXXXX69]", "[XXXXX70]", "[XXXXX71]", "[XXXXX72]", "[XXXXX73]", "[XXXXX74]", "[XXXXX75]", "[XXXXX76]", "[XXXXX77]", "[XXXXX78]", "[XXXXX79]", "[XXXXX80]", "[XXXXX81]", "[XXXXX82]", "[XXXXX83]", "[XXXXX84]", "[XXXXX85]", "[XXXXX86]", "[XXXXX87]", "[XXXXX88]", "[XXXXX89]", "[XXXXX90]", "[XXXXX91]", "[XXXXX92]", "[XXXXX93]", "[XXXXX94]", "[XXXXX95]", "[XXXXX96]", "[XXXXX97]", "[XXXXX98]", "[XXXXX99]", "[XXXXX100]", "[XXXXX101]", "[XXXXX102]", "[XXXXX103]", "[XXXXX104]", "[XXXXX105]", "[XXXXX106]", "[XXXXX107]", "[XXXXX108]", "[XXXXX109]", "[XXXXX110]", "[XXXXX111]", "[XXXXX112]", "[XXXXX113]", "[XXXXX114]", "[XXXXX115]", "[XXXXX116]", "[XXXXX117]", "[XXXXX118]", "[XXXXX119]", "[XXXXX120]", "[XXXXX121]", "[XXXXX122]", "[XXXXX123]", "[XXXXX124]", "[XXXXX125]", "[XXXXX126]", "[XXXXX127]", "[XXXXX128]", "[XXXXX129]", "[XXXXX130]", "[XXXXX131]", "[XXXXX132]", "[XXXXX133]", "[XXXXX134]", "[XXXXX135]", "[XXXXX136]", "[XXXXX137]", "[XXXXX138]", "[XXXXX139]", "[XXXXX140]", "[XXXXX141]", "[XXXXX142]", "[XXXXX143]", "[XXXXX144]", "[XXXXX145]", "[XXXXX146]", "[XXXXX147]", "[XXXXX148]", "[XXXXX149]", "[XXXXX150]", "[XXXXX151]", "[XXXXX152]", "[XXXXX153]", "[XXXXX154]", "[XXXXX155]", "[XXXXX156]", "[XXXXX157]", "[XXXXX158]", "[XXXXX159]", "[XXXXX160]", "[XXXXX161]", "[XXXXX162]", "[XXXXX163]", "[XXXXX164]", "[XXXXX165]", "[XXXXX166]", "[XXXXX167]", "[XXXXX168]", "[XXXXX169]", "[XXXXX170]", "[XXXXX171]", "[XXXXX172]", "[XXXXX173]", "[XXXXX174]", "[XXXXX175]", "[XXXXX176]", "[XXXXX177]", "[XXXXX178]", "[XXXXX179]", "[XXXXX180]", "[XXXXX181]", "[XXXXX182]", "[XXXXX183]", "[XXXXX184]", "[XXXXX185]", "[XXXXX186]", "[XXXXX187]", "[XXXXX188]", "[XXXXX189]", "[XXXXX190]", "[XXXXX191]", "[XXXXX192]", "[XXXXX193]", "[XXXXX194]", "[XXXXX195]", "[XXXXX196]", "[XXXXX197]", "[XXXXX198]", "[XXXXX199]", "[XXXXX200]", "[XXXXX201]", "[XXXXX202]", "[XXXXX203]", "[XXXXX204]", "[XXXXX205]", "[XXXXX206]", "[XXXXX207]", "[XXXXX208]", "[XXXXX209]", "[XXXXX210]", "[XXXXX211]", "[XXXXX212]", "[XXXXX213]", "[XXXXX214]", "[XXXXX215]", "[XXXXX216]", "[XXXXX217]", "[XXXXX218]", "[XXXXX219]", "[XXXXX220]", "[XXXXX221]", "[XXXXX222]", "[XXXXX223]", "[XXXXX224]", "[XXXXX225]", "[XXXXX226]", "[XXXXX227]", "[XXXXX228]", "[XXXXX229]", "[XXXXX230]", "[XXXXX231]", "[XXXXX232]", "[XXXXX233]", "[XXXXX234]", "[XXXXX235]", "[XXXXX236]", "[XXXXX237]", "[XXXXX238]", "[XXXXX239]", "[XXXXX240]", "[XXXXX241]", "[XXXXX242]", "[XXXXX243]", "[XXXXX244]", "[XXXXX245]", "[XXXXX246]", "[XXXXX247]", "[XXXXX248]", "[XXXXX249]", "[XXXXX250]", "[XXXXX251]", "[XXXXX252]", "[XXXXX253]", "[XXXXX254]", "[XXXXX255]", "[XXXXX256]", "[XXXXX257]", "[XXXXX258]", "[XXXXX259]", "[XXXXX260]", "[XXXXX261]", "[XXXXX262]", "[XXXXX263]", "[XXXXX264]", "[XXXXX265]", "[XXXXX266]", "[XXXXX267]", "[XXXXX268]", "[XXXXX269]", "[XXXXX270]", "[XXXXX271]", "[XXXXX272]", "[XXXXX273]", "[XXXXX274]", "[XXXXX275]", "[XXXXX276]", "[XXXXX277]", "[XXXXX278]", "[XXXXX279]", "[XXXXX280]", "[XXXXX281]", "[XXXXX282]", "[XXXXX283]", "[XXXXX284]", "[XXXXX285]", "[XXXXX286]", "[XXXXX287]", "[XXXXX288]", "[XXXXX289]", "[XXXXX290]", "[XXXXX291]", "[XXXXX292]", "[XXXXX293]", "[XXXXX294]", "[XXXXX295]", "[XXXXX296]", "[XXXXX297]", "[XXXXX298]", "[XXXXX299]", "[XXXXX300]", "[XXXXX301]", "[XXXXX302]", "[XXXXX303]", "[XXXXX304]", "[XXXXX305]", "[XXXXX306]", "[XXXXX307]", "[XXXXX308]", "[XXXXX309]", "[XXXXX310]", "[XXXXX311]", "[XXXXX312]", "[XXXXX313]", "[XXXXX314]", "[XXXXX315]", "[XXXXX316]", "[XXXXX317]", "[XXXXX318]", "[XXXXX319]", "[XXXXX320]", "[XXXXX321]", "[XXXXX322]", "[XXXXX323]", "[XXXXX324]", "[XXXXX325]", "[XXXXX326]", "[XXXXX327]", "[XXXXX328]", "[XXXXX329]", "[XXXXX330]", "[XXXXX331]", "[XXXXX332]", "[XXXXX333]", "[XXXXX334]", "[XXXXX335]", "[XXXXX336]", "[XXXXX337]", "[XXXXX338]", "[XXXXX339]", "[XXXXX340]", "[XXXXX341]", "[XXXXX342]", "[XXXXX343]", "[XXXXX344]", "[XXXXX345]", "[XXXXX346]", "[XXXXX347]", "[XXXXX348]", "[XXXXX349]", "[XXXXX350]", "[XXXXX351]", "[XXXXX352]", "[XXXXX353]", "[XXXXX354]", "[XXXXX355]", "[XXXXX356]", "[XXXXX357]", "[XXXXX358]", "[XXXXX359]", "[XXXXX360]", "[XXXXX361]", "[XXXXX362]", "[XXXXX363]", "[XXXXX364]", "[XXXXX365]", "[XXXXX366]", "[XXXXX367]", "[XXXXX368]", "[XXXXX369]", "[XXXXX370]", "[XXXXX371]", "[XXXXX372]", "[XXXXX373]", "[XXXXX374]", "[XXXXX375]", "[XXXXX376]", "[XXXXX377]", "[XXXXX378]", "[XXXXX379]", "[XXXXX380]", "[XXXXX381]", "[XXXXX382]", "[XXXXX383]", "[XXXXX384]", "[XXXXX385]", "[XXXXX386]", "[XXXXX387]", "[XXXXX388]", "[XXXXX389]", "[XXXXX390]", "[XXXXX391]", "[XXXXX392]", "[XXXXX393]", "[XXXXX394]", "[XXXXX395]", "[XXXXX396]", "[XXXXX397]", "[XXXXX398]", "[XXXXX399]", "[XXXXX400]", "[XXXXX401]", "[XXXXX402]", "[XXXXX403]", "[XXXXX404]", "[XXXXX405]", "[XXXXX406]", "[XXXXX407]", "[XXXXX408]", "[XXXXX409]", "[XXXXX410]", "[XXXXX411]", "[XXXXX412]", "[XXXXX413]", "[XXXXX414]", "[XXXXX415]", "[XXXXX416]", "[XXXXX417]", "[XXXXX418]", "[XXXXX419]", "[XXXXX420]", "[XXXXX421]", "[XXXXX422]", "[XXXXX423]", "[XXXXX424]", "[XXXXX425]", "[XXXXX426]", "[XXXXX427]", "[XXXXX428]", "[XXXXX429]", "[XXXXX430]", "[XXXXX431]", "[XXXXX432]", "[XXXXX433]", "[XXXXX434]", "[XXXXX435]", "[XXXXX436]", "[XXXXX437]", "[XXXXX438]", "[XXXXX439]", "[XXXXX440]", "[XXXXX441]", "[XXXXX442]", "[XXXXX443]", "[XXXXX444]", "[XXXXX445]", "[XXXXX446]", "[XXXXX447]", "[XXXXX448]", "[XXXXX449]", "[XXXXX450]", "[XXXXX451]", "[XXXXX452]", "[XXXXX453]", "[XXXXX454]", "[XXXXX455]", "[XXXXX456]", "[XXXXX457]", "[XXXXX458]", "[XXXXX459]", "[XXXXX460]", "[XXXXX461]", "[XXXXX462]", "[XXXXX463]", "[XXXXX464]", "[XXXXX465]", "[XXXXX466]", "[XXXXX467]", "[XXXXX468]", "[XXXXX469]", "[XXXXX470]", "[XXXXX471]", "[XXXXX472]", "[XXXXX473]", "[XXXXX474]", "[XXXXX475]", "[XXXXX476]", "[XXXXX477]", "[XXXXX478]", "[XXXXX479]", "[XXXXX480]", "[XXXXX481]", "[XXXXX482]", "[XXXXX483]", "[XXXXX484]", "[XXXXX485]", "[XXXXX486]", "[XXXXX487]", "[XXXXX488]", "[XXXXX489]", "[XXXXX490]", "[XXXXX491]", "[XXXXX492]", "[XXXXX493]", "[XXXXX494]", "[XXXXX495]", "[XXXXX496]", "[XXXXX497]", "[XXXXX498]", "[XXXXX499]", "[XXXXX500]", "[XXXXX501]", "[XXXXX502]", "[XXXXX503]", "[XXXXX504]", "[XXXXX505]", "[XXXXX506]", "[XXXXX507]", "[XXXXX508]", "[XXXXX509]", "[XXXXX510]", "[XXXXX511]", "[XXXXX512]", "[XXXXX513]", "[XXXXX514]", "[XXXXX515]", "[XXXXX516]", "[XXXXX517]", "[XXXXX518]", "[XXXXX519]", "[XXXXX520]", "[XXXXX521]", "[XXXXX522]", "[XXXXX523]", "[XXXXX524]", "[XXXXX525]", "[XXXXX526]", "[XXXXX527]", "[XXXXX528]", "[XXXXX529]", "[XXXXX530]", "[XXXXX531]", "[XXXXX532]", "[XXXXX533]", "[XXXXX534]", "[XXXXX535]", "[XXXXX536]", "[XXXXX537]", "[XXXXX538]", "[XXXXX539]", "[XXXXX540]", "[XXXXX541]", "[XXXXX542]", "[XXXXX543]", "[XXXXX544]", "[XXXXX545]", "[XXXXX546]", "[XXXXX547]", "[XXXXX548]", "[XXXXX549]", "[XXXXX550]", "[XXXXX551]", "[XXXXX552]", "[XXXXX553]", "[XXXXX554]", "[XXXXX555]", "[XXXXX556]", "[XXXXX557]", "[XXXXX558]", "[XXXXX559]", "[XXXXX560]", "[XXXXX561]", "[XXXXX562]", "[XXXXX563]", "[XXXXX564]", "[XXXXX565]", "[XXXXX566]", "[XXXXX567]", "[XXXXX568]", "[XXXXX569]", "[XXXXX570]", "[XXXXX571]", "[XXXXX572]", "[XXXXX573]", "[XXXXX574]", "[XXXXX575]", "[XXXXX576]", "[XXXXX577]", "[XXXXX578]", "[XXXXX579]", "[XXXXX580]", "[XXXXX581]", "[XXXXX582]", "[XXXXX583]", "[XXXXX584]", "[XXXXX585]", "[XXXXX586]", "[XXXXX587]", "[XXXXX588]", "[XXXXX589]", "[XXXXX590]", "[XXXXX591]", "[XXXXX592]", "[XXXXX593]", "[XXXXX594]", "[XXXXX595]", "[XXXXX596]", "[XXXXX597]", "[XXXXX598]", "[XXXXX599]", "[XXXXX600]", "[XXXXX601]", "[XXXXX602]", "[XXXXX603]", "[XXXXX604]", "[XXXXX605]", "[XXXXX606]", "[XXXXX607]", "[XXXXX608]", "[XXXXX609]", "[XXXXX610]", "[XXXXX611]", "[XXXXX612]", "[XXXXX613]", "[XXXXX614]", "[XXXXX615]", "[XXXXX616]", "[XXXXX617]", "[XXXXX618]", "[XXXXX619]", "[XXXXX620]", "[XXXXX621]", "[XXXXX622]", "[XXXXX623]", "[XXXXX624]", "[XXXXX625]", "[XXXXX626]", "[XXXXX627]", "[XXXXX628]", "[XXXXX629]", "[XXXXX630]", "[XXXXX631]", "[XXXXX632]", "[XXXXX633]", "[XXXXX634]", "[XXXXX635]", "[XXXXX636]", "[XXXXX637]", "[XXXXX638]", "[XXXXX639]", "[XXXXX640]", "[XXXXX641]", "[XXXXX642]", "[XXXXX643]", "[XXXXX644]", "[XXXXX645]", "[XXXXX646]", "[XXXXX647]", "[XXXXX648]", "[XXXXX649]", "[XXXXX650]", "[XXXXX651]", "[XXXXX652]", "[XXXXX653]", "[XXXXX654]", "[XXXXX655]", "[XXXXX656]", "[XXXXX657]", "[XXXXX658]", "[XXXXX659]", "[XXXXX660]", "[XXXXX661]", "[XXXXX662]", "[XXXXX663]", "[XXXXX664]", "[XXXXX665]", "[XXXXX666]", "[XXXXX667]", "[XXXXX668]", "[XXXXX669]", "[XXXXX670]", "[XXXXX671]", "[XXXXX672]", "[XXXXX673]", "[XXXXX674]", "[XXXXX675]", "[XXXXX676]", "[XXXXX677]", "[XXXXX678]", "[XXXXX679]", "[XXXXX680]", "[XXXXX681]", "[XXXXX682]", "[XXXXX683]", "[XXXXX684]", "[XXXXX685]", "[XXXXX686]", "[XXXXX687]", "[XXXXX688]", "[XXXXX689]", "[XXXXX690]", "[XXXXX691]", "[XXXXX692]", "[XXXXX693]", "[XXXXX694]", "[XXXXX695]", "[XXXXX696]", "[XXXXX697]", "[XXXXX698]", "[XXXXX699]", "[XXXXX700]", "[XXXXX701]", "[XXXXX702]", "[XXXXX703]", "[XXXXX704]", "[XXXXX705]", "[XXXXX706]", "[XXXXX707]", "[XXXXX708]", "[XXXXX709]", "[XXXXX710]", "[XXXXX711]", "[XXXXX712]", "[XXXXX713]", "[XXXXX714]", "[XXXXX715]", "[XXXXX716]", "[XXXXX717]", "[XXXXX718]", "[XXXXX719]", "[XXXXX720]", "[XXXXX721]", "[XXXXX722]", "[XXXXX723]", "[XXXXX724]", "[XXXXX725]", "[XXXXX726]", "[XXXXX727]", "[XXXXX728]", "[XXXXX729]", "[XXXXX730]", "[XXXXX731]", "[XXXXX732]", "[XXXXX733]", "[XXXXX734]", "[XXXXX735]", "[XXXXX736]", "[XXXXX737]", "[XXXXX738]", "[XXXXX739]", "[XXXXX740]", "[XXXXX741]", "[XXXXX742]", "[XXXXX743]", "[XXXXX744]", "[XXXXX745]", "[XXXXX746]", "[XXXXX747]", "[XXXXX748]", "[XXXXX749]", "[XXXXX750]", "[XXXXX751]", "[XXXXX752]", "[XXXXX753]", "[XXXXX754]", "[XXXXX755]", "[XXXXX756]", "[XXXXX757]", "[XXXXX758]", "[XXXXX759]", "[XXXXX760]", "[XXXXX761]", "[XXXXX762]", "[XXXXX763]", "[XXXXX764]", "[XXXXX765]", "[XXXXX766]", "[XXXXX767]", "[XXXXX768]", "[XXXXX769]", "[XXXXX770]", "[XXXXX771]", "[XXXXX772]", "[XXXXX773]", "[XXXXX774]", "[XXXXX775]", "[XXXXX776]", "[XXXXX777]", "[XXXXX778]", "[XXXXX779]", "[XXXXX780]", "[XXXXX781]", "[XXXXX782]", "[XXXXX783]", "[XXXXX784]", "[XXXXX785]", "[XXXXX786]", "[XXXXX787]", "[XXXXX788]", "[XXXXX789]", "[XXXXX790]", "[XXXXX791]", "[XXXXX792]", "[XXXXX793]", "[XXXXX794]", "[XXXXX795]", "[XXXXX796]", "[XXXXX797]", "[XXXXX798]", "[XXXXX799]", "[XXXXX800]", "[XXXXX801]", "[XXXXX802]", "[XXXXX803]", "[XXXXX804]", "[XXXXX805]", "[XXXXX806]", "[XXXXX807]", "[XXXXX808]", "[XXXXX809]", "[XXXXX810]", "[XXXXX811]", "[XXXXX812]", "[XXXXX813]", "[XXXXX814]", "[XXXXX815]", "[XXXXX816]", "[XXXXX817]", "[XXXXX818]", "[XXXXX819]", "[XXXXX820]", "[XXXXX821]", "[XXXXX822]", "[XXXXX823]", "[XXXXX824]", "[XXXXX825]", "[XXXXX826]", "[XXXXX827]", "[XXXXX828]", "[XXXXX829]", "[XXXXX830]", "[XXXXX831]", "[XXXXX832]", "[XXXXX833]", "[XXXXX834]", "[XXXXX835]", "[XXXXX836]", "[XXXXX837]", "[XXXXX838]", "[XXXXX839]", "[XXXXX840]", "[XXXXX841]", "[XXXXX842]", "[XXXXX843]", "[XXXXX844]", "[XXXXX845]", "[XXXXX846]", "[XXXXX847]", "[XXXXX848]", "[XXXXX849]", "[XXXXX850]", "[XXXXX851]", "[XXXXX852]", "[XXXXX853]", "[XXXXX854]", "[XXXXX855]", "[XXXXX856]", "[XXXXX857]", "[XXXXX858]", "[XXXXX859]", "[XXXXX860]", "[XXXXX861]", "[XXXXX862]", "[XXXXX863]", "[XXXXX864]", "[XXXXX865]", "[XXXXX866]", "[XXXXX867]", "[XXXXX868]", "[XXXXX869]", "[XXXXX870]", "[XXXXX871]", "[XXXXX872]", "[XXXXX873]", "[XXXXX874]", "[XXXXX875]", "[XXXXX876]", "[XXXXX877]", "[XXXXX878]", "[XXXXX879]", "[XXXXX880]", "[XXXXX881]", "[XXXXX882]", "[XXXXX883]", "[XXXXX884]", "[XXXXX885]", "[XXXXX886]", "[XXXXX887]", "[XXXXX888]", "[XXXXX889]", "[XXXXX890]", "[XXXXX891]", "[XXXXX892]", "[XXXXX893]", "[XXXXX894]", "[XXXXX895]", "[XXXXX896]", "[XXXXX897]", "[XXXXX898]", "[XXXXX899]", "[XXXXX900]", "[XXXXX901]", "[XXXXX902]", "[XXXXX903]", "[XXXXX904]", "[XXXXX905]", "[XXXXX906]", "[XXXXX907]", "[XXXXX908]", "[XXXXX909]", "[XXXXX910]", "[XXXXX911]", "[XXXXX912]", "[XXXXX913]", "[XXXXX914]", "[XXXXX915]", "[XXXXX916]", "[XXXXX917]", "[XXXXX918]", "[XXXXX919]", "[XXXXX920]", "[XXXXX921]", "[XXXXX922]", "[XXXXX923]", "[XXXXX924]", "[XXXXX925]", "[XXXXX926]", "[XXXXX927]", "[XXXXX928]", "[XXXXX929]", "[XXXXX930]", "[XXXXX931]", "[XXXXX932]", "[XXXXX933]", "[XXXXX934]", "[XXXXX935]", "[XXXXX936]", "[XXXXX937]", "[XXXXX938]", "[XXXXX939]", "[XXXXX940]", "[XXXXX941]", "[XXXXX942]", "[XXXXX943]", "[XXXXX944]", "[XXXXX945]", "[XXXXX946]", "[XXXXX947]", "[XXXXX948]", "[XXXXX949]", "[XXXXX950]", "[XXXXX951]", "[XXXXX952]", "[XXXXX953]", "[XXXXX954]", "[XXXXX955]", "[XXXXX956]", "[XXXXX957]", "[XXXXX958]", "[XXXXX959]", "[XXXXX960]", "[XXXXX961]", "[XXXXX962]", "[XXXXX963]", "[XXXXX964]", "[XXXXX965]", "[XXXXX966]", "[XXXXX967]", "[XXXXX968]", "[XXXXX969]", "[XXXXX970]", "[XXXXX971]", "[XXXXX972]", "[XXXXX973]", "[XXXXX974]", "[XXXXX975]", "[XXXXX976]", "[XXXXX977]", "[XXXXX978]", "[XXXXX979]", "[XXXXX980]", "[XXXXX981]", "[XXXXX982]", "[XXXXX983]", "[XXXXX984]", "[XXXXX985]", "[XXXXX986]", "[XXXXX987]", "[XXXXX988]", "[XXXXX989]", "[XXXXX990]", "[XXXXX991]", "[XXXXX992]", "[XXXXX993]", "[XXXXX994]", "[XXXXX995]", "[XXXXX996]", "[XXXXX997]", "[XXXXX998]", "[XXXXX999]", "[XXXXX1000]", "[XXXXX1001]", "[XXXXX1002]", "[XXXXX1003]", "[XXXXX1004]", "[XXXXX1005]", "[XXXXX1006]", "[XXXXX1007]", "[XXXXX1008]", "[XXXXX1009]", "[XXXXX1010]", "[XXXXX1011]", "[XXXXX1012]", "[XXXXX1013]", "[XXXXX1014]", "[XXXXX1015]", "[XXXXX1016]", "[XXXXX1017]", "[XXXXX1018]", "[XXXXX1019]", "[XXXXX1020]", "[XXXXX1021]", "[XXXXX1022]", "[XXXXX1023]", "[XXXXX1024]", "[XXXXX1025]", "[XXXXX1026]", "[XXXXX1027]", "[XXXXX1028]", "[XXXXX1029]", "[XXXXX1030]", "[XXXXX1031]", "[XXXXX1032]", "[XXXXX1033]", "[XXXXX1034]", "[XXXXX1035]", "[XXXXX1036]", "[XXXXX1037]", "[XXXXX1038]", "[XXXXX1039]", "[XXXXX1040]", "[XXXXX1041]", "[XXXXX1042]", "[XXXXX1043]", "[XXXXX1044]", "[XXXXX1045]", "[XXXXX1046]", "[XXXXX1047]", "[XXXXX1048]", "[XXXXX1049]", "[XXXXX1050]", "[XXXXX1051]", "[XXXXX1052]", "[XXXXX1053]", "[XXXXX1054]", "[XXXXX1055]", "[XXXXX1056]", "[XXXXX1057]", "[XXXXX1058]", "[XXXXX1059]", "[XXXXX1060]", "[XXXXX1061]", "[XXXXX1062]", "[XXXXX1063]", "[XXXXX1064]", "[XXXXX1065]", "[XXXXX1066]", "[XXXXX1067]", "[XXXXX1068]", "[XXXXX1069]", "[XXXXX1070]", "[XXXXX1071]", "[XXXXX1072]", "[XXXXX1073]", "[XXXXX1074]", "[XXXXX1075]", "[XXXXX1076]", "[XXXXX1077]", "[XXXXX1078]", "[XXXXX1079]", "[XXXXX1080]", "[XXXXX1081]", "[XXXXX1082]", "[XXXXX1083]", "[XXXXX1084]", "[XXXXX1085]", "[XXXXX1086]", "[XXXXX1087]", "[XXXXX1088]", "[XXXXX1089]", "[XXXXX1090]", "[XXXXX1091]", "[XXXXX1092]", "[XXXXX1093]", "[XXXXX1094]", "[XXXXX1095]", "[XXXXX1096]", "[XXXXX1097]", "[XXXXX1098]", "[XXXXX1099]", "[XXXXX1100]", "[XXXXX1101]", "[XXXXX1102]", "[XXXXX1103]", "[XXXXX1104]", "[XXXXX1105]", "[XXXXX1106]", "[XXXXX1107]", "[XXXXX1108]", "[XXXXX1109]", "[XXXXX1110]", "[XXXXX1111]", "[XXXXX1112]", "[XXXXX1113]", "[XXXXX1114]", "[XXXXX1115]", "[XXXXX1116]", "[XXXXX1117]", "[XXXXX1118]", "[XXXXX1119]", "[XXXXX1120]", "[XXXXX1121]", "[XXXXX1122]", "[XXXXX1123]", "[XXXXX1124]", "[XXXXX1125]", "[XXXXX1126]", "[XXXXX1127]", "[XXXXX1128]", "[XXXXX1129]", "[XXXXX1130]", "[XXXXX1131]", "[XXXXX1132]", "[XXXXX1133]", "[XXXXX1134]", "[XXXXX1135]", "[XXXXX1136]", "[XXXXX1137]", "[XXXXX1138]", "[XXXXX1139]", "[XXXXX1140]", "[XXXXX1141]", "[XXXXX1142]", "[XXXXX1143]", "[XXXXX1144]", "[XXXXX1145]", "[XXXXX1146]", "[XXXXX1147]", "[XXXXX1148]", "[XXXXX1149]", "[XXXXX1150]", "[XXXXX1151]", "[XXXXX1152]", "[XXXXX1153]", "[XXXXX1154]", "[XXXXX1155]", "[XXXXX1156]", "[XXXXX1157]", "[XXXXX1158]", "[XXXXX1159]", "[XXXXX1160]", "[XXXXX1161]", "[XXXXX1162]", "[XXXXX1163]", "[XXXXX1164]", "[XXXXX1165]", "[XXXXX1166]", "[XXXXX1167]", "[XXXXX1168]", "[XXXXX1169]", "[XXXXX1170]", "[XXXXX1171]", "[XXXXX1172]", "[XXXXX1173]", "[XXXXX1174]", "[XXXXX1175]", "[XXXXX1176]", "[XXXXX1177]", "[XXXXX1178]", "[XXXXX1179]", "[XXXXX1180]", "[XXXXX1181]", "[XXXXX1182]", "[XXXXX1183]", "[XXXXX1184]", "[XXXXX1185]", "[XXXXX1186]", "[XXXXX1187]", "[XXXXX1188]", "[XXXXX1189]", "[XXXXX1190]", "[XXXXX1191]", "[XXXXX1192]", "[XXXXX1193]", "[XXXXX1194]", "[XXXXX1195]", "[XXXXX1196]", "[XXXXX1197]", "[XXXXX1198]", "[XXXXX1199]", "[XXXXX1200]", "[XXXXX1201]", "[XXXXX1202]", "[XXXXX1203]", "[XXXXX1204]", "[XXXXX1205]", "[XXXXX1206]", "[XXXXX1207]", "[XXXXX1208]", "[XXXXX1209]", "[XXXXX1210]", "[XXXXX1211]", "[XXXXX1212]", "[XXXXX1213]", "[XXXXX1214]", "[XXXXX1215]", "[XXXXX1216]", "[XXXXX1217]", "[XXXXX1218]", "[XXXXX1219]", "[XXXXX1220]", "[XXXXX1221]", "[XXXXX1222]", "[XXXXX1223]", "[XXXXX1224]", "[XXXXX1225]", "[XXXXX1226]", "[XXXXX1227]", "[XXXXX1228]", "[XXXXX1229]", "[XXXXX1230]", "[XXXXX1231]", "[XXXXX1232]", "[XXXXX1233]", "[XXXXX1234]", "[XXXXX1235]"]}
spiece.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:01ed39a765168f9bf60f9c8a7e075db26ccec9b448aaef02b7dfe94292102a51
3
+ size 676115
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"do_lower_case": false, "remove_space": true, "keep_accents": true, "bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "<unk>", "sep_token": "[SEP]", "pad_token": "<pad>", "cls_token": "[CLS]", "mask_token": {"content": "[MASK]", "single_word": false, "lstrip": true, "rstrip": false, "normalized": false, "__type": "AddedToken"}, "sp_model_kwargs": {}, "name_or_path": "models/full/srn_latn_full", "model_input_names": ["input_ids", "attention_mask"], "special_tokens_map_file": "models/full/srn_latn_full/special_tokens_map.json", "tokenizer_class": "AlbertTokenizer"}