Upload bgp_latn_5mb tokenizer.

added_tokens.json

@@ -0,0 +1 @@
+{"[XXXXX485]": 44667, "[XXXXX312]": 44494, "[XXXXX134]": 44316, "[XXXXX70]": 44252, "[XXXXX505]": 44687, "[XXXXX827]": 45009, "[XXXXX417]": 44599, "[XXXXX403]": 44585, "[XXXXX475]": 44657, "[XXXXX810]": 44992, "[XXXXX172]": 44354, "[XXXXX711]": 44893, "[XXXXX404]": 44586, "[XXXXX668]": 44850, "[XXXXX740]": 44922, "[XXXXX110]": 44292, "[XXXXX833]": 45015, "[XXXXX846]": 45028, "[XXXXX247]": 44429, "[XXXXX256]": 44438, "[XXXXX440]": 44622, "[XXXXX709]": 44891, "[XXXXX620]": 44802, "[XXXXX566]": 44748, "[XXXXX6]": 44188, "[XXXXX724]": 44906, "[XXXXX437]": 44619, "[XXXXX126]": 44308, "[XXXXX869]": 45051, "[XXXXX571]": 44753, "[XXXXX817]": 44999, "[XXXXX602]": 44784, "[XXXXX603]": 44785, "[XXXXX531]": 44713, "[XXXXX732]": 44914, "[XXXXX598]": 44780, "[XXXXX153]": 44335, "[XXXXX63]": 44245, "[XXXXX319]": 44501, "[XXXXX122]": 44304, "[XXXXX128]": 44310, "[SEP]": 44179, "[XXXXX873]": 45055, "[XXXXX478]": 44660, "[XXXXX559]": 44741, "[XXXXX555]": 44737, "[XXXXX611]": 44793, "[XXXXX609]": 44791, "[XXXXX726]": 44908, "[XXXXX811]": 44993, "[XXXXX245]": 44427, "[XXXXX47]": 44229, "[XXXXX384]": 44566, "[XXXXX294]": 44476, "[XXXXX91]": 44273, "[XXXXX434]": 44616, "[XXXXX104]": 44286, "[XXXXX358]": 44540, "[XXXXX265]": 44447, "[XXXXX462]": 44644, "[XXXXX704]": 44886, "[XXXXX93]": 44275, "[XXXXX745]": 44927, "[XXXXX343]": 44525, "[XXXXX693]": 44875, "[XXXXX4]": 44186, "[XXXXX281]": 44463, "[XXXXX141]": 44323, "[XXXXX64]": 44246, "[XXXXX378]": 44560, "[XXXXX101]": 44283, "[XXXXX32]": 44214, "[XXXXX50]": 44232, "[XXXXX107]": 44289, "[XXXXX218]": 44400, "[XXXXX801]": 44983, "[XXXXX36]": 44218, "[XXXXX246]": 44428, "[XXXXX798]": 44980, "[XXXXX10]": 44192, "[XXXXX29]": 44211, "[XXXXX857]": 45039, "[XXXXX171]": 44353, "[XXXXX365]": 44547, "[XXXXX130]": 44312, "[XXXXX267]": 44449, "[XXXXX55]": 44237, "[XXXXX739]": 44921, "[XXXXX89]": 44271, "[XXXXX229]": 44411, "[XXXXX0]": 44182, "[XXXXX150]": 44332, "[XXXXX66]": 44248, "[XXXXX264]": 44446, "[XXXXX375]": 44557, "[XXXXX850]": 45032, "[XXXXX781]": 44963, "[XXXXX203]": 44385, "[XXXXX772]": 44954, "[XXXXX277]": 44459, "[XXXXX111]": 44293, "[XXXXX815]": 44997, "[XXXXX108]": 44290, "[XXXXX376]": 44558, "[XXXXX595]": 44777, "[XXXXX200]": 44382, "[XXXXX30]": 44212, "[XXXXX304]": 44486, "[XXXXX439]": 44621, "[XXXXX257]": 44439, "[XXXXX249]": 44431, "[XXXXX621]": 44803, "[XXXXX647]": 44829, "[XXXXX232]": 44414, "[XXXXX550]": 44732, "[XXXXX317]": 44499, "[XXXXX695]": 44877, "[XXXXX433]": 44615, "[XXXXX368]": 44550, "[XXXXX583]": 44765, "[XXXXX752]": 44934, "[XXXXX839]": 45021, "[XXXXX79]": 44261, "[XXXXX155]": 44337, "[XXXXX99]": 44281, "[XXXXX612]": 44794, "[XXXXX654]": 44836, "[XXXXX283]": 44465, "[XXXXX388]": 44570, "[XXXXX701]": 44883, "[XXXXX533]": 44715, "[XXXXX46]": 44228, "[XXXXX427]": 44609, "[XXXXX511]": 44693, "[XXXXX198]": 44380, "[XXXXX789]": 44971, "[XXXXX280]": 44462, "[XXXXX357]": 44539, "[XXXXX285]": 44467, "[XXXXX529]": 44711, "[XXXXX335]": 44517, "[XXXXX273]": 44455, "[XXXXX858]": 45040, "[XXXXX596]": 44778, "[XXXXX761]": 44943, "[XXXXX542]": 44724, "[XXXXX823]": 45005, "[XXXXX831]": 45013, "[XXXXX809]": 44991, "[XXXXX390]": 44572, "[XXXXX487]": 44669, "[XXXXX588]": 44770, "[XXXXX103]": 44285, "[XXXXX848]": 45030, "[XXXXX838]": 45020, "[XXXXX344]": 44526, "[XXXXX371]": 44553, "[XXXXX405]": 44587, "[XXXXX315]": 44497, "[XXXXX715]": 44897, "[XXXXX493]": 44675, "[XXXXX34]": 44216, "[XXXXX502]": 44684, "[XXXXX824]": 45006, "[XXXXX771]": 44953, "[XXXXX512]": 44694, "[XXXXX734]": 44916, "[XXXXX510]": 44692, "[XXXXX498]": 44680, "[XXXXX676]": 44858, "[XXXXX398]": 44580, "[XXXXX42]": 44224, "[XXXXX124]": 44306, "[XXXXX737]": 44919, "[XXXXX686]": 44868, "[XXXXX225]": 44407, "[XXXXX213]": 44395, "[XXXXX325]": 44507, "[XXXXX287]": 44469, "[XXXXX170]": 44352, "[XXXXX193]": 44375, "[XXXXX482]": 44664, "[XXXXX692]": 44874, "[XXXXX190]": 44372, "[XXXXX196]": 44378, "[XXXXX780]": 44962, "[XXXXX146]": 44328, "[XXXXX820]": 45002, "[XXXXX345]": 44527, "[XXXXX12]": 44194, "[XXXXX472]": 44654, "[XXXXX211]": 44393, "[XXXXX832]": 45014, "[XXXXX100]": 44282, "[XXXXX735]": 44917, "[XXXXX276]": 44458, "[XXXXX646]": 44828, "[XXXXX349]": 44531, "[XXXXX658]": 44840, "[XXXXX138]": 44320, "[XXXXX492]": 44674, "[XXXXX230]": 44412, "[XXXXX342]": 44524, "[XXXXX477]": 44659, "[XXXXX635]": 44817, "[XXXXX793]": 44975, "[XXXXX540]": 44722, "[XXXXX259]": 44441, "[XXXXX195]": 44377, "[XXXXX698]": 44880, "[XXXXX26]": 44208, "[XXXXX469]": 44651, "[XXXXX84]": 44266, "[XXXXX804]": 44986, "[XXXXX3]": 44185, "[XXXXX685]": 44867, "[XXXXX159]": 44341, "[XXXXX237]": 44419, "[XXXXX854]": 45036, "[XXXXX638]": 44820, "[XXXXX713]": 44895, "[XXXXX473]": 44655, "[XXXXX744]": 44926, "[XXXXX574]": 44756, "[XXXXX775]": 44957, "[XXXXX428]": 44610, "[XXXXX800]": 44982, "[XXXXX291]": 44473, "[XXXXX518]": 44700, "[XXXXX562]": 44744, "[XXXXX393]": 44575, "[XXXXX690]": 44872, "[XXXXX133]": 44315, "[XXXXX435]": 44617, "[XXXXX725]": 44907, "[XXXXX514]": 44696, "[XXXXX834]": 45016, "[XXXXX167]": 44349, "[XXXXX872]": 45054, "[XXXXX169]": 44351, "[XXXXX563]": 44745, "[XXXXX154]": 44336, "[XXXXX292]": 44474, "[XXXXX353]": 44535, "[XXXXX484]": 44666, "[XXXXX28]": 44210, "[XXXXX180]": 44362, "[XXXXX825]": 45007, "[XXXXX597]": 44779, "[XXXXX60]": 44242, "[XXXXX145]": 44327, "[XXXXX135]": 44317, "[XXXXX841]": 45023, "[XXXXX221]": 44403, "[XXXXX396]": 44578, "[XXXXX836]": 45018, "[XXXXX148]": 44330, "[XXXXX286]": 44468, "[XXXXX522]": 44704, "[XXXXX62]": 44244, "[XXXXX627]": 44809, "[XXXXX68]": 44250, "[XXXXX501]": 44683, "[XXXXX45]": 44227, "[XXXXX253]": 44435, "[XXXXX109]": 44291, "[XXXXX426]": 44608, "[XXXXX431]": 44613, "[XXXXX759]": 44941, "[XXXXX640]": 44822, "[XXXXX40]": 44222, "[XXXXX260]": 44442, "[XXXXX425]": 44607, "[XXXXX290]": 44472, "[XXXXX447]": 44629, "[XXXXX822]": 45004, "[XXXXX528]": 44710, "[XXXXX359]": 44541, "[XXXXX694]": 44876, "[XXXXX61]": 44243, "[XXXXX455]": 44637, "[XXXXX136]": 44318, "[XXXXX769]": 44951, "[XXXXX614]": 44796, "[XXXXX316]": 44498, "[XXXXX446]": 44628, "[XXXXX224]": 44406, "[XXXXX391]": 44573, "[XXXXX251]": 44433, "[XXXXX389]": 44571, "[XXXXX669]": 44851, "[XXXXX27]": 44209, "[XXXXX383]": 44565, "[XXXXX643]": 44825, "[XXXXX202]": 44384, "[XXXXX212]": 44394, "[XXXXX5]": 44187, "[XXXXX346]": 44528, "[XXXXX71]": 44253, "[XXXXX210]": 44392, "[XXXXX380]": 44562, "[XXXXX121]": 44303, "[XXXXX272]": 44454, "[XXXXX387]": 44569, "[XXXXX500]": 44682, "[XXXXX401]": 44583, "[XXXXX803]": 44985, "[XXXXX590]": 44772, "[XXXXX457]": 44639, "[XXXXX185]": 44367, "[XXXXX279]": 44461, "[XXXXX480]": 44662, "[XXXXX717]": 44899, "[XXXXX651]": 44833, "[XXXXX667]": 44849, "[XXXXX239]": 44421, "[XXXXX194]": 44376, "[XXXXX164]": 44346, "[XXXXX240]": 44422, "[XXXXX367]": 44549, "[XXXXX779]": 44961, "[XXXXX189]": 44371, "[XXXXX82]": 44264, "[XXXXX576]": 44758, "[XXXXX795]": 44977, "[XXXXX90]": 44272, "[XXXXX161]": 44343, "[XXXXX814]": 44996, "[XXXXX449]": 44631, "[XXXXX364]": 44546, "[XXXXX722]": 44904, "[XXXXX17]": 44199, "[XXXXX303]": 44485, "[XXXXX11]": 44193, "[XXXXX112]": 44294, "[XXXXX258]": 44440, "[XXXXX168]": 44350, "[XXXXX684]": 44866, "[XXXXX227]": 44409, "[XXXXX67]": 44249, "[XXXXX305]": 44487, "[XXXXX608]": 44790, "[XXXXX289]": 44471, "[XXXXX22]": 44204, "[XXXXX73]": 44255, "[XXXXX7]": 44189, "[XXXXX152]": 44334, "[XXXXX244]": 44426, "[XXXXX25]": 44207, "[XXXXX407]": 44589, "[XXXXX360]": 44542, "[XXXXX379]": 44561, "[XXXXX720]": 44902, "[XXXXX14]": 44196, "[MASK]": 44181, "[XXXXX468]": 44650, "[XXXXX302]": 44484, "[CLS]": 44178, "[XXXXX470]": 44652, "[XXXXX731]": 44913, "[XXXXX460]": 44642, "[XXXXX556]": 44738, "[XXXXX770]": 44952, "[XXXXX142]": 44324, "[XXXXX826]": 45008, "[XXXXX790]": 44972, "[XXXXX413]": 44595, "[XXXXX373]": 44555, "[XXXXX339]": 44521, "[XXXXX592]": 44774, "[XXXXX444]": 44626, "[XXXXX114]": 44296, "[XXXXX222]": 44404, "[XXXXX311]": 44493, "[XXXXX648]": 44830, "[XXXXX424]": 44606, "[XXXXX516]": 44698, "[XXXXX288]": 44470, "[XXXXX641]": 44823, "[XXXXX631]": 44813, "[XXXXX429]": 44611, "[XXXXX102]": 44284, "[XXXXX295]": 44477, "[XXXXX216]": 44398, "[XXXXX113]": 44295, "[XXXXX489]": 44671, "[XXXXX2]": 44184, "[XXXXX377]": 44559, "[XXXXX324]": 44506, "[XXXXX802]": 44984, "[XXXXX828]": 45010, "[XXXXX610]": 44792, "[XXXXX747]": 44929, "[XXXXX191]": 44373, "[XXXXX707]": 44889, "[XXXXX72]": 44254, "[XXXXX532]": 44714, "[XXXXX575]": 44757, "[XXXXX830]": 45012, "[XXXXX399]": 44581, "[XXXXX653]": 44835, "[XXXXX600]": 44782, "[XXXXX509]": 44691, "[XXXXX306]": 44488, "[XXXXX118]": 44300, "[XXXXX650]": 44832, "[XXXXX791]": 44973, "[XXXXX459]": 44641, "[XXXXX682]": 44864, "[XXXXX120]": 44302, "[XXXXX845]": 45027, "[XXXXX657]": 44839, "[XXXXX441]": 44623, "[XXXXX537]": 44719, "[XXXXX644]": 44826, "[XXXXX284]": 44466, "[XXXXX503]": 44685, "[XXXXX636]": 44818, "[XXXXX422]": 44604, "[XXXXX92]": 44274, "[XXXXX623]": 44805, "[XXXXX599]": 44781, "[XXXXX119]": 44301, "[XXXXX454]": 44636, "[XXXXX51]": 44233, "[XXXXX749]": 44931, "[XXXXX773]": 44955, "[XXXXX1]": 44183, "[XXXXX626]": 44808, "[XXXXX250]": 44432, "[XXXXX331]": 44513, "[XXXXX792]": 44974, "[XXXXX741]": 44923, "[XXXXX785]": 44967, "[XXXXX123]": 44305, "[XXXXX217]": 44399, "[XXXXX751]": 44933, "[XXXXX624]": 44806, "[XXXXX524]": 44706, "[XXXXX179]": 44361, "[XXXXX499]": 44681, "[XXXXX340]": 44522, "[XXXXX96]": 44278, "[XXXXX341]": 44523, "[XXXXX421]": 44603, "[XXXXX842]": 45024, "[XXXXX508]": 44690, "[XXXXX206]": 44388, "[XXXXX763]": 44945, "[XXXXX151]": 44333, "[XXXXX716]": 44898, "[XXXXX400]": 44582, "[XXXXX86]": 44268, "[XXXXX9]": 44191, "[XXXXX699]": 44881, "[XXXXX463]": 44645, "[XXXXX821]": 45003, "[XXXXX274]": 44456, "[XXXXX252]": 44434, "[XXXXX736]": 44918, "[XXXXX567]": 44749, "[XXXXX33]": 44215, "[XXXXX402]": 44584, "[XXXXX573]": 44755, "[XXXXX601]": 44783, "[XXXXX868]": 45050, "[XXXXX506]": 44688, "[XXXXX670]": 44852, "[XXXXX496]": 44678, "[XXXXX835]": 45017, "[XXXXX415]": 44597, "[XXXXX395]": 44577, "[XXXXX548]": 44730, "[XXXXX705]": 44887, "[XXXXX23]": 44205, "[XXXXX406]": 44588, "[XXXXX410]": 44592, "[XXXXX125]": 44307, "[XXXXX88]": 44270, "[XXXXX94]": 44276, "[XXXXX618]": 44800, "[XXXXX544]": 44726, "[XXXXX215]": 44397, "[XXXXX436]": 44618, "[XXXXX630]": 44812, "[XXXXX712]": 44894, "[XXXXX517]": 44699, "[XXXXX642]": 44824, "[XXXXX205]": 44387, "[XXXXX553]": 44735, "[XXXXX703]": 44885, "[XXXXX677]": 44859, "[XXXXX54]": 44236, "[XXXXX656]": 44838, "[XXXXX420]": 44602, "[XXXXX721]": 44903, "[XXXXX774]": 44956, "[XXXXX786]": 44968, "[XXXXX35]": 44217, "[XXXXX782]": 44964, "[XXXXX551]": 44733, "[XXXXX625]": 44807, "[XXXXX241]": 44423, "[XXXXX738]": 44920, "[XXXXX266]": 44448, "[XXXXX689]": 44871, "[XXXXX536]": 44718, "[XXXXX156]": 44338, "[XXXXX851]": 45033, "[XXXXX649]": 44831, "[XXXXX53]": 44235, "[XXXXX688]": 44870, "[XXXXX363]": 44545, "[XXXXX615]": 44797, "[XXXXX829]": 45011, "[XXXXX564]": 44746, "[XXXXX843]": 45025, "[XXXXX149]": 44331, "[XXXXX728]": 44910, "[XXXXX59]": 44241, "[XXXXX856]": 45038, "[XXXXX855]": 45037, "[XXXXX374]": 44556, "[XXXXX691]": 44873, "[XXXXX777]": 44959, "[XXXXX706]": 44888, "[XXXXX708]": 44890, "[XXXXX796]": 44978, "[XXXXX526]": 44708, "[XXXXX613]": 44795, "[XXXXX263]": 44445, "[XXXXX450]": 44632, "[XXXXX719]": 44901, "[XXXXX697]": 44879, "[XXXXX591]": 44773, "[XXXXX481]": 44663, "[XXXXX77]": 44259, "[XXXXX318]": 44500, "[XXXXX840]": 45022, "[XXXXX19]": 44201, "[XXXXX453]": 44635, "[XXXXX619]": 44801, "[XXXXX672]": 44854, "[XXXXX362]": 44544, "[XXXXX483]": 44665, "[XXXXX15]": 44197, "[XXXXX226]": 44408, "[XXXXX197]": 44379, "[XXXXX799]": 44981, "[XXXXX606]": 44788, "[XXXXX776]": 44958, "[XXXXX535]": 44717, "[XXXXX547]": 44729, "[XXXXX214]": 44396, "[XXXXX750]": 44932, "[XXXXX192]": 44374, "[XXXXX558]": 44740, "[XXXXX561]": 44743, "[XXXXX662]": 44844, "[XXXXX607]": 44789, "[XXXXX645]": 44827, "[XXXXX356]": 44538, "[XXXXX557]": 44739, "[XXXXX837]": 45019, "[XXXXX207]": 44389, "[XXXXX632]": 44814, "[XXXXX546]": 44728, "[XXXXX764]": 44946, "[XXXXX300]": 44482, "[XXXXX163]": 44345, "[XXXXX589]": 44771, "[XXXXX479]": 44661, "[XXXXX680]": 44862, "[XXXXX818]": 45000, "[XXXXX355]": 44537, "[XXXXX115]": 44297, "[XXXXX471]": 44653, "[XXXXX350]": 44532, "[XXXXX560]": 44742, "[XXXXX38]": 44220, "[XXXXX204]": 44386, "[XXXXX348]": 44530, "[XXXXX549]": 44731, "[XXXXX853]": 45035, "[XXXXX314]": 44496, "[XXXXX397]": 44579, "[XXXXX81]": 44263, "[XXXXX659]": 44841, "[XXXXX637]": 44819, "[XXXXX127]": 44309, "[XXXXX755]": 44937, "[XXXXX530]": 44712, "[XXXXX243]": 44425, "[XXXXX816]": 44998, "[XXXXX381]": 44563, "[XXXXX710]": 44892, "[XXXXX859]": 45041, "[XXXXX334]": 44516, "[XXXXX729]": 44911, "[XXXXX673]": 44855, "[XXXXX412]": 44594, "[XXXXX753]": 44935, "[XXXXX756]": 44938, "[XXXXX593]": 44775, "[XXXXX768]": 44950, "[XXXXX678]": 44860, "[XXXXX534]": 44716, "[XXXXX794]": 44976, "[XXXXX188]": 44370, "[XXXXX604]": 44786, "[XXXXX31]": 44213, "[XXXXX605]": 44787, "[XXXXX467]": 44649, "[XXXXX418]": 44600, "[XXXXX748]": 44930, "[XXXXX209]": 44391, "[XXXXX321]": 44503, "[XXXXX788]": 44970, "[XXXXX297]": 44479, "[XXXXX448]": 44630, "[XXXXX231]": 44413, "[XXXXX742]": 44924, "[XXXXX49]": 44231, "[XXXXX269]": 44451, "[XXXXX497]": 44679, "[XXXXX666]": 44848, "[XXXXX385]": 44567, "[XXXXX139]": 44321, "[XXXXX860]": 45042, "[XXXXX616]": 44798, "[XXXXX332]": 44514, "[XXXXX328]": 44510, "[XXXXX271]": 44453, "[XXXXX797]": 44979, "[XXXXX458]": 44640, "[XXXXX83]": 44265, "[XXXXX541]": 44723, "[XXXXX354]": 44536, "[XXXXX106]": 44288, "[XXXXX58]": 44240, "[XXXXX733]": 44915, "[XXXXX812]": 44994, "[XXXXX143]": 44325, "[XXXXX466]": 44648, "[XXXXX622]": 44804, "[XXXXX451]": 44633, "[XXXXX97]": 44279, "[XXXXX254]": 44436, "[XXXXX336]": 44518, "[XXXXX579]": 44761, "[XXXXX844]": 45026, "[XXXXX465]": 44647, "[XXXXX629]": 44811, "[XXXXX675]": 44857, "[XXXXX569]": 44751, "[XXXXX762]": 44944, "[XXXXX870]": 45052, "[XXXXX323]": 44505, "[XXXXX48]": 44230, "[XXXXX633]": 44815, "<pad>": 44180, "[XXXXX617]": 44799, "[XXXXX663]": 44845, "[XXXXX523]": 44705, "[XXXXX730]": 44912, "[XXXXX248]": 44430, "[XXXXX219]": 44401, "[XXXXX147]": 44329, "[XXXXX849]": 45031, "[XXXXX327]": 44509, "[XXXXX861]": 45043, "[XXXXX766]": 44948, "[XXXXX652]": 44834, "[XXXXX320]": 44502, "[XXXXX65]": 44247, "[XXXXX568]": 44750, "[XXXXX309]": 44491, "[XXXXX57]": 44239, "[XXXXX177]": 44359, "[XXXXX235]": 44417, "[XXXXX700]": 44882, "[XXXXX293]": 44475, "[XXXXX386]": 44568, "[XXXXX461]": 44643, "[XXXXX382]": 44564, "[XXXXX369]": 44551, "[XXXXX310]": 44492, "[XXXXX488]": 44670, "[XXXXX95]": 44277, "[XXXXX865]": 45047, "[XXXXX20]": 44202, "[XXXXX201]": 44383, "[XXXXX238]": 44420, "[XXXXX714]": 44896, "[XXXXX234]": 44416, "[XXXXX452]": 44634, "[XXXXX173]": 44355, "[XXXXX158]": 44340, "[XXXXX44]": 44226, "[XXXXX464]": 44646, "[XXXXX866]": 45048, "[XXXXX39]": 44221, "[XXXXX137]": 44319, "[XXXXX754]": 44936, "[XXXXX702]": 44884, "[XXXXX181]": 44363, "[XXXXX322]": 44504, "[XXXXX525]": 44707, "[XXXXX76]": 44258, "[XXXXX41]": 44223, "[XXXXX787]": 44969, "[XXXXX140]": 44322, "[XXXXX131]": 44313, "[XXXXX233]": 44415, "[XXXXX664]": 44846, "[XXXXX223]": 44405, "[XXXXX98]": 44280, "[XXXXX847]": 45029, "[XXXXX807]": 44989, "[XXXXX411]": 44593, "[XXXXX129]": 44311, "[XXXXX805]": 44987, "[XXXXX299]": 44481, "[XXXXX521]": 44703, "[XXXXX495]": 44677, "[XXXXX438]": 44620, "[XXXXX539]": 44721, "[XXXXX490]": 44672, "[XXXXX160]": 44342, "[XXXXX308]": 44490, "[XXXXX639]": 44821, "[XXXXX871]": 45053, "[XXXXX242]": 44424, "[XXXXX671]": 44853, "[XXXXX746]": 44928, "[XXXXX414]": 44596, "[XXXXX347]": 44529, "[XXXXX270]": 44452, "[XXXXX52]": 44234, "[XXXXX456]": 44638, "[XXXXX361]": 44543, "[XXXXX174]": 44356, "[XXXXX582]": 44764, "[XXXXX275]": 44457, "[XXXXX116]": 44298, "[XXXXX8]": 44190, "[XXXXX337]": 44519, "[XXXXX208]": 44390, "[XXXXX783]": 44965, "[XXXXX187]": 44369, "[XXXXX767]": 44949, "[XXXXX75]": 44257, "[XXXXX333]": 44515, "[XXXXX696]": 44878, "[XXXXX117]": 44299, "[XXXXX545]": 44727, "[XXXXX570]": 44752, "[XXXXX166]": 44348, "[XXXXX432]": 44614, "[XXXXX727]": 44909, "[XXXXX278]": 44460, "[XXXXX105]": 44287, "[XXXXX445]": 44627, "[XXXXX301]": 44483, "[XXXXX504]": 44686, "[XXXXX679]": 44861, "[XXXXX681]": 44863, "[XXXXX85]": 44267, "[XXXXX184]": 44366, "[XXXXX552]": 44734, "[XXXXX366]": 44548, "[XXXXX175]": 44357, "[XXXXX329]": 44511, "[XXXXX370]": 44552, "[XXXXX581]": 44763, "[XXXXX78]": 44260, "[XXXXX687]": 44869, "[XXXXX183]": 44365, "[XXXXX594]": 44776, "[XXXXX419]": 44601, "[XXXXX199]": 44381, "[XXXXX228]": 44410, "[XXXXX182]": 44364, "[XXXXX351]": 44533, "[XXXXX723]": 44905, "[XXXXX157]": 44339, "[XXXXX515]": 44697, "[XXXXX543]": 44725, "[XXXXX520]": 44702, "[XXXXX674]": 44856, "[XXXXX867]": 45049, "[XXXXX813]": 44995, "[XXXXX808]": 44990, "[XXXXX743]": 44925, "[XXXXX282]": 44464, "[XXXXX330]": 44512, "[XXXXX655]": 44837, "[XXXXX16]": 44198, "[XXXXX476]": 44658, "[XXXXX87]": 44269, "[XXXXX144]": 44326, "[XXXXX430]": 44612, "[XXXXX43]": 44225, "[XXXXX408]": 44590, "[XXXXX554]": 44736, "[XXXXX313]": 44495, "[XXXXX486]": 44668, "[XXXXX18]": 44200, "[XXXXX186]": 44368, "[XXXXX409]": 44591, "[XXXXX565]": 44747, "[XXXXX578]": 44760, "[XXXXX580]": 44762, "[XXXXX74]": 44256, "[XXXXX298]": 44480, "[XXXXX758]": 44940, "[XXXXX37]": 44219, "[XXXXX628]": 44810, "[XXXXX665]": 44847, "[XXXXX784]": 44966, "[XXXXX443]": 44625, "[XXXXX718]": 44900, "[XXXXX491]": 44673, "[XXXXX132]": 44314, "[XXXXX862]": 45044, "[XXXXX372]": 44554, "[XXXXX519]": 44701, "[XXXXX220]": 44402, "[XXXXX819]": 45001, "[XXXXX394]": 44576, "[XXXXX165]": 44347, "[XXXXX13]": 44195, "[XXXXX852]": 45034, "[XXXXX21]": 44203, "[XXXXX261]": 44443, "[XXXXX24]": 44206, "[XXXXX236]": 44418, "[XXXXX863]": 45045, "[XXXXX69]": 44251, "[XXXXX757]": 44939, "[XXXXX765]": 44947, "[XXXXX416]": 44598, "[XXXXX338]": 44520, "[XXXXX494]": 44676, "[XXXXX176]": 44358, "[XXXXX307]": 44489, "[XXXXX586]": 44768, "[XXXXX634]": 44816, "[XXXXX584]": 44766, "[XXXXX778]": 44960, "[XXXXX661]": 44843, "[XXXXX162]": 44344, "[XXXXX442]": 44624, "[XXXXX572]": 44754, "[XXXXX683]": 44865, "[XXXXX178]": 44360, "[XXXXX474]": 44656, "[XXXXX392]": 44574, "[XXXXX513]": 44695, "[XXXXX423]": 44605, "[XXXXX56]": 44238, "[XXXXX538]": 44720, "[XXXXX255]": 44437, "[XXXXX352]": 44534, "[XXXXX660]": 44842, "[XXXXX527]": 44709, "[XXXXX585]": 44767, "[XXXXX268]": 44450, "[XXXXX262]": 44444, "[XXXXX806]": 44988, "[XXXXX326]": 44508, "[XXXXX296]": 44478, "[XXXXX507]": 44689, "[XXXXX587]": 44769, "[XXXXX577]": 44759, "[XXXXX760]": 44942, "[XXXXX864]": 45046, "[XXXXX80]": 44262}

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "<unk>", "sep_token": "[SEP]", "pad_token": "<pad>", "cls_token": "[CLS]", "mask_token": {"content": "[MASK]", "single_word": false, "lstrip": true, "rstrip": false, "normalized": false}, "additional_special_tokens": ["[XXXXX0]", "[XXXXX1]", "[XXXXX2]", "[XXXXX3]", "[XXXXX4]", "[XXXXX5]", "[XXXXX6]", "[XXXXX7]", "[XXXXX8]", "[XXXXX9]", "[XXXXX10]", "[XXXXX11]", "[XXXXX12]", "[XXXXX13]", "[XXXXX14]", "[XXXXX15]", "[XXXXX16]", "[XXXXX17]", "[XXXXX18]", "[XXXXX19]", "[XXXXX20]", "[XXXXX21]", "[XXXXX22]", "[XXXXX23]", "[XXXXX24]", "[XXXXX25]", "[XXXXX26]", "[XXXXX27]", "[XXXXX28]", "[XXXXX29]", "[XXXXX30]", "[XXXXX31]", "[XXXXX32]", "[XXXXX33]", "[XXXXX34]", "[XXXXX35]", "[XXXXX36]", "[XXXXX37]", "[XXXXX38]", "[XXXXX39]", "[XXXXX40]", "[XXXXX41]", "[XXXXX42]", "[XXXXX43]", "[XXXXX44]", "[XXXXX45]", "[XXXXX46]", "[XXXXX47]", "[XXXXX48]", "[XXXXX49]", "[XXXXX50]", "[XXXXX51]", "[XXXXX52]", "[XXXXX53]", "[XXXXX54]", "[XXXXX55]", "[XXXXX56]", "[XXXXX57]", "[XXXXX58]", "[XXXXX59]", "[XXXXX60]", "[XXXXX61]", "[XXXXX62]", "[XXXXX63]", "[XXXXX64]", "[XXXXX65]", "[XXXXX66]", "[XXXXX67]", "[XXXXX68]", "[XXXXX69]", "[XXXXX70]", "[XXXXX71]", "[XXXXX72]", "[XXXXX73]", "[XXXXX74]", "[XXXXX75]", "[XXXXX76]", "[XXXXX77]", "[XXXXX78]", "[XXXXX79]", "[XXXXX80]", "[XXXXX81]", "[XXXXX82]", "[XXXXX83]", "[XXXXX84]", "[XXXXX85]", "[XXXXX86]", "[XXXXX87]", "[XXXXX88]", "[XXXXX89]", "[XXXXX90]", "[XXXXX91]", "[XXXXX92]", "[XXXXX93]", "[XXXXX94]", "[XXXXX95]", "[XXXXX96]", "[XXXXX97]", "[XXXXX98]", "[XXXXX99]", "[XXXXX100]", "[XXXXX101]", "[XXXXX102]", "[XXXXX103]", "[XXXXX104]", "[XXXXX105]", "[XXXXX106]", "[XXXXX107]", "[XXXXX108]", "[XXXXX109]", "[XXXXX110]", "[XXXXX111]", "[XXXXX112]", "[XXXXX113]", "[XXXXX114]", "[XXXXX115]", "[XXXXX116]", "[XXXXX117]", "[XXXXX118]", "[XXXXX119]", "[XXXXX120]", "[XXXXX121]", "[XXXXX122]", "[XXXXX123]", "[XXXXX124]", "[XXXXX125]", "[XXXXX126]", "[XXXXX127]", "[XXXXX128]", "[XXXXX129]", "[XXXXX130]", "[XXXXX131]", "[XXXXX132]", "[XXXXX133]", "[XXXXX134]", "[XXXXX135]", "[XXXXX136]", "[XXXXX137]", "[XXXXX138]", "[XXXXX139]", "[XXXXX140]", "[XXXXX141]", "[XXXXX142]", "[XXXXX143]", "[XXXXX144]", "[XXXXX145]", "[XXXXX146]", "[XXXXX147]", "[XXXXX148]", "[XXXXX149]", "[XXXXX150]", "[XXXXX151]", "[XXXXX152]", "[XXXXX153]", "[XXXXX154]", "[XXXXX155]", "[XXXXX156]", "[XXXXX157]", "[XXXXX158]", "[XXXXX159]", "[XXXXX160]", "[XXXXX161]", "[XXXXX162]", "[XXXXX163]", "[XXXXX164]", "[XXXXX165]", "[XXXXX166]", "[XXXXX167]", "[XXXXX168]", "[XXXXX169]", "[XXXXX170]", "[XXXXX171]", "[XXXXX172]", "[XXXXX173]", "[XXXXX174]", "[XXXXX175]", "[XXXXX176]", "[XXXXX177]", "[XXXXX178]", "[XXXXX179]", "[XXXXX180]", "[XXXXX181]", "[XXXXX182]", "[XXXXX183]", "[XXXXX184]", "[XXXXX185]", "[XXXXX186]", "[XXXXX187]", "[XXXXX188]", "[XXXXX189]", "[XXXXX190]", "[XXXXX191]", "[XXXXX192]", "[XXXXX193]", "[XXXXX194]", "[XXXXX195]", "[XXXXX196]", "[XXXXX197]", "[XXXXX198]", "[XXXXX199]", "[XXXXX200]", "[XXXXX201]", "[XXXXX202]", "[XXXXX203]", "[XXXXX204]", "[XXXXX205]", "[XXXXX206]", "[XXXXX207]", "[XXXXX208]", "[XXXXX209]", "[XXXXX210]", "[XXXXX211]", "[XXXXX212]", "[XXXXX213]", "[XXXXX214]", "[XXXXX215]", "[XXXXX216]", "[XXXXX217]", "[XXXXX218]", "[XXXXX219]", "[XXXXX220]", "[XXXXX221]", "[XXXXX222]", "[XXXXX223]", "[XXXXX224]", "[XXXXX225]", "[XXXXX226]", "[XXXXX227]", "[XXXXX228]", "[XXXXX229]", "[XXXXX230]", "[XXXXX231]", "[XXXXX232]", "[XXXXX233]", "[XXXXX234]", "[XXXXX235]", "[XXXXX236]", "[XXXXX237]", "[XXXXX238]", "[XXXXX239]", "[XXXXX240]", "[XXXXX241]", "[XXXXX242]", "[XXXXX243]", "[XXXXX244]", "[XXXXX245]", "[XXXXX246]", "[XXXXX247]", "[XXXXX248]", "[XXXXX249]", "[XXXXX250]", "[XXXXX251]", "[XXXXX252]", "[XXXXX253]", "[XXXXX254]", "[XXXXX255]", "[XXXXX256]", "[XXXXX257]", "[XXXXX258]", "[XXXXX259]", "[XXXXX260]", "[XXXXX261]", "[XXXXX262]", "[XXXXX263]", "[XXXXX264]", "[XXXXX265]", "[XXXXX266]", "[XXXXX267]", "[XXXXX268]", "[XXXXX269]", "[XXXXX270]", "[XXXXX271]", "[XXXXX272]", "[XXXXX273]", "[XXXXX274]", "[XXXXX275]", "[XXXXX276]", "[XXXXX277]", "[XXXXX278]", "[XXXXX279]", "[XXXXX280]", "[XXXXX281]", "[XXXXX282]", "[XXXXX283]", "[XXXXX284]", "[XXXXX285]", "[XXXXX286]", "[XXXXX287]", "[XXXXX288]", "[XXXXX289]", "[XXXXX290]", "[XXXXX291]", "[XXXXX292]", "[XXXXX293]", "[XXXXX294]", "[XXXXX295]", "[XXXXX296]", "[XXXXX297]", "[XXXXX298]", "[XXXXX299]", "[XXXXX300]", "[XXXXX301]", "[XXXXX302]", "[XXXXX303]", "[XXXXX304]", "[XXXXX305]", "[XXXXX306]", "[XXXXX307]", "[XXXXX308]", "[XXXXX309]", "[XXXXX310]", "[XXXXX311]", "[XXXXX312]", "[XXXXX313]", "[XXXXX314]", "[XXXXX315]", "[XXXXX316]", "[XXXXX317]", "[XXXXX318]", "[XXXXX319]", "[XXXXX320]", "[XXXXX321]", "[XXXXX322]", "[XXXXX323]", "[XXXXX324]", "[XXXXX325]", "[XXXXX326]", "[XXXXX327]", "[XXXXX328]", "[XXXXX329]", "[XXXXX330]", "[XXXXX331]", "[XXXXX332]", "[XXXXX333]", "[XXXXX334]", "[XXXXX335]", "[XXXXX336]", "[XXXXX337]", "[XXXXX338]", "[XXXXX339]", "[XXXXX340]", "[XXXXX341]", "[XXXXX342]", "[XXXXX343]", "[XXXXX344]", "[XXXXX345]", "[XXXXX346]", "[XXXXX347]", "[XXXXX348]", "[XXXXX349]", "[XXXXX350]", "[XXXXX351]", "[XXXXX352]", "[XXXXX353]", "[XXXXX354]", "[XXXXX355]", "[XXXXX356]", "[XXXXX357]", "[XXXXX358]", "[XXXXX359]", "[XXXXX360]", "[XXXXX361]", "[XXXXX362]", "[XXXXX363]", "[XXXXX364]", "[XXXXX365]", "[XXXXX366]", "[XXXXX367]", "[XXXXX368]", "[XXXXX369]", "[XXXXX370]", "[XXXXX371]", "[XXXXX372]", "[XXXXX373]", "[XXXXX374]", "[XXXXX375]", "[XXXXX376]", "[XXXXX377]", "[XXXXX378]", "[XXXXX379]", "[XXXXX380]", "[XXXXX381]", "[XXXXX382]", "[XXXXX383]", "[XXXXX384]", "[XXXXX385]", "[XXXXX386]", "[XXXXX387]", "[XXXXX388]", "[XXXXX389]", "[XXXXX390]", "[XXXXX391]", "[XXXXX392]", "[XXXXX393]", "[XXXXX394]", "[XXXXX395]", "[XXXXX396]", "[XXXXX397]", "[XXXXX398]", "[XXXXX399]", "[XXXXX400]", "[XXXXX401]", "[XXXXX402]", "[XXXXX403]", "[XXXXX404]", "[XXXXX405]", "[XXXXX406]", "[XXXXX407]", "[XXXXX408]", "[XXXXX409]", "[XXXXX410]", "[XXXXX411]", "[XXXXX412]", "[XXXXX413]", "[XXXXX414]", "[XXXXX415]", "[XXXXX416]", "[XXXXX417]", "[XXXXX418]", "[XXXXX419]", "[XXXXX420]", "[XXXXX421]", "[XXXXX422]", "[XXXXX423]", "[XXXXX424]", "[XXXXX425]", "[XXXXX426]", "[XXXXX427]", "[XXXXX428]", "[XXXXX429]", "[XXXXX430]", "[XXXXX431]", "[XXXXX432]", "[XXXXX433]", "[XXXXX434]", "[XXXXX435]", "[XXXXX436]", "[XXXXX437]", "[XXXXX438]", "[XXXXX439]", "[XXXXX440]", "[XXXXX441]", "[XXXXX442]", "[XXXXX443]", "[XXXXX444]", "[XXXXX445]", "[XXXXX446]", "[XXXXX447]", "[XXXXX448]", "[XXXXX449]", "[XXXXX450]", "[XXXXX451]", "[XXXXX452]", "[XXXXX453]", "[XXXXX454]", "[XXXXX455]", "[XXXXX456]", "[XXXXX457]", "[XXXXX458]", "[XXXXX459]", "[XXXXX460]", "[XXXXX461]", "[XXXXX462]", "[XXXXX463]", "[XXXXX464]", "[XXXXX465]", "[XXXXX466]", "[XXXXX467]", "[XXXXX468]", "[XXXXX469]", "[XXXXX470]", "[XXXXX471]", "[XXXXX472]", "[XXXXX473]", "[XXXXX474]", "[XXXXX475]", "[XXXXX476]", "[XXXXX477]", "[XXXXX478]", "[XXXXX479]", "[XXXXX480]", "[XXXXX481]", "[XXXXX482]", "[XXXXX483]", "[XXXXX484]", "[XXXXX485]", "[XXXXX486]", "[XXXXX487]", "[XXXXX488]", "[XXXXX489]", "[XXXXX490]", "[XXXXX491]", "[XXXXX492]", "[XXXXX493]", "[XXXXX494]", "[XXXXX495]", "[XXXXX496]", "[XXXXX497]", "[XXXXX498]", "[XXXXX499]", "[XXXXX500]", "[XXXXX501]", "[XXXXX502]", "[XXXXX503]", "[XXXXX504]", "[XXXXX505]", "[XXXXX506]", "[XXXXX507]", "[XXXXX508]", "[XXXXX509]", "[XXXXX510]", "[XXXXX511]", "[XXXXX512]", "[XXXXX513]", "[XXXXX514]", "[XXXXX515]", "[XXXXX516]", "[XXXXX517]", "[XXXXX518]", "[XXXXX519]", "[XXXXX520]", "[XXXXX521]", "[XXXXX522]", "[XXXXX523]", "[XXXXX524]", "[XXXXX525]", "[XXXXX526]", "[XXXXX527]", "[XXXXX528]", "[XXXXX529]", "[XXXXX530]", "[XXXXX531]", "[XXXXX532]", "[XXXXX533]", "[XXXXX534]", "[XXXXX535]", "[XXXXX536]", "[XXXXX537]", "[XXXXX538]", "[XXXXX539]", "[XXXXX540]", "[XXXXX541]", "[XXXXX542]", "[XXXXX543]", "[XXXXX544]", "[XXXXX545]", "[XXXXX546]", "[XXXXX547]", "[XXXXX548]", "[XXXXX549]", "[XXXXX550]", "[XXXXX551]", "[XXXXX552]", "[XXXXX553]", "[XXXXX554]", "[XXXXX555]", "[XXXXX556]", "[XXXXX557]", "[XXXXX558]", "[XXXXX559]", "[XXXXX560]", "[XXXXX561]", "[XXXXX562]", "[XXXXX563]", "[XXXXX564]", "[XXXXX565]", "[XXXXX566]", "[XXXXX567]", "[XXXXX568]", "[XXXXX569]", "[XXXXX570]", "[XXXXX571]", "[XXXXX572]", "[XXXXX573]", "[XXXXX574]", "[XXXXX575]", "[XXXXX576]", "[XXXXX577]", "[XXXXX578]", "[XXXXX579]", "[XXXXX580]", "[XXXXX581]", "[XXXXX582]", "[XXXXX583]", "[XXXXX584]", "[XXXXX585]", "[XXXXX586]", "[XXXXX587]", "[XXXXX588]", "[XXXXX589]", "[XXXXX590]", "[XXXXX591]", "[XXXXX592]", "[XXXXX593]", "[XXXXX594]", "[XXXXX595]", "[XXXXX596]", "[XXXXX597]", "[XXXXX598]", "[XXXXX599]", "[XXXXX600]", "[XXXXX601]", "[XXXXX602]", "[XXXXX603]", "[XXXXX604]", "[XXXXX605]", "[XXXXX606]", "[XXXXX607]", "[XXXXX608]", "[XXXXX609]", "[XXXXX610]", "[XXXXX611]", "[XXXXX612]", "[XXXXX613]", "[XXXXX614]", "[XXXXX615]", "[XXXXX616]", "[XXXXX617]", "[XXXXX618]", "[XXXXX619]", "[XXXXX620]", "[XXXXX621]", "[XXXXX622]", "[XXXXX623]", "[XXXXX624]", "[XXXXX625]", "[XXXXX626]", "[XXXXX627]", "[XXXXX628]", "[XXXXX629]", "[XXXXX630]", "[XXXXX631]", "[XXXXX632]", "[XXXXX633]", "[XXXXX634]", "[XXXXX635]", "[XXXXX636]", "[XXXXX637]", "[XXXXX638]", "[XXXXX639]", "[XXXXX640]", "[XXXXX641]", "[XXXXX642]", "[XXXXX643]", "[XXXXX644]", "[XXXXX645]", "[XXXXX646]", "[XXXXX647]", "[XXXXX648]", "[XXXXX649]", "[XXXXX650]", "[XXXXX651]", "[XXXXX652]", "[XXXXX653]", "[XXXXX654]", "[XXXXX655]", "[XXXXX656]", "[XXXXX657]", "[XXXXX658]", "[XXXXX659]", "[XXXXX660]", "[XXXXX661]", "[XXXXX662]", "[XXXXX663]", "[XXXXX664]", "[XXXXX665]", "[XXXXX666]", "[XXXXX667]", "[XXXXX668]", "[XXXXX669]", "[XXXXX670]", "[XXXXX671]", "[XXXXX672]", "[XXXXX673]", "[XXXXX674]", "[XXXXX675]", "[XXXXX676]", "[XXXXX677]", "[XXXXX678]", "[XXXXX679]", "[XXXXX680]", "[XXXXX681]", "[XXXXX682]", "[XXXXX683]", "[XXXXX684]", "[XXXXX685]", "[XXXXX686]", "[XXXXX687]", "[XXXXX688]", "[XXXXX689]", "[XXXXX690]", "[XXXXX691]", "[XXXXX692]", "[XXXXX693]", "[XXXXX694]", "[XXXXX695]", "[XXXXX696]", "[XXXXX697]", "[XXXXX698]", "[XXXXX699]", "[XXXXX700]", "[XXXXX701]", "[XXXXX702]", "[XXXXX703]", "[XXXXX704]", "[XXXXX705]", "[XXXXX706]", "[XXXXX707]", "[XXXXX708]", "[XXXXX709]", "[XXXXX710]", "[XXXXX711]", "[XXXXX712]", "[XXXXX713]", "[XXXXX714]", "[XXXXX715]", "[XXXXX716]", "[XXXXX717]", "[XXXXX718]", "[XXXXX719]", "[XXXXX720]", "[XXXXX721]", "[XXXXX722]", "[XXXXX723]", "[XXXXX724]", "[XXXXX725]", "[XXXXX726]", "[XXXXX727]", "[XXXXX728]", "[XXXXX729]", "[XXXXX730]", "[XXXXX731]", "[XXXXX732]", "[XXXXX733]", "[XXXXX734]", "[XXXXX735]", "[XXXXX736]", "[XXXXX737]", "[XXXXX738]", "[XXXXX739]", "[XXXXX740]", "[XXXXX741]", "[XXXXX742]", "[XXXXX743]", "[XXXXX744]", "[XXXXX745]", "[XXXXX746]", "[XXXXX747]", "[XXXXX748]", "[XXXXX749]", "[XXXXX750]", "[XXXXX751]", "[XXXXX752]", "[XXXXX753]", "[XXXXX754]", "[XXXXX755]", "[XXXXX756]", "[XXXXX757]", "[XXXXX758]", "[XXXXX759]", "[XXXXX760]", "[XXXXX761]", "[XXXXX762]", "[XXXXX763]", "[XXXXX764]", "[XXXXX765]", "[XXXXX766]", "[XXXXX767]", "[XXXXX768]", "[XXXXX769]", "[XXXXX770]", "[XXXXX771]", "[XXXXX772]", "[XXXXX773]", "[XXXXX774]", "[XXXXX775]", "[XXXXX776]", "[XXXXX777]", "[XXXXX778]", "[XXXXX779]", "[XXXXX780]", "[XXXXX781]", "[XXXXX782]", "[XXXXX783]", "[XXXXX784]", "[XXXXX785]", "[XXXXX786]", "[XXXXX787]", "[XXXXX788]", "[XXXXX789]", "[XXXXX790]", "[XXXXX791]", "[XXXXX792]", "[XXXXX793]", "[XXXXX794]", "[XXXXX795]", "[XXXXX796]", "[XXXXX797]", "[XXXXX798]", "[XXXXX799]", "[XXXXX800]", "[XXXXX801]", "[XXXXX802]", "[XXXXX803]", "[XXXXX804]", "[XXXXX805]", "[XXXXX806]", "[XXXXX807]", "[XXXXX808]", "[XXXXX809]", "[XXXXX810]", "[XXXXX811]", "[XXXXX812]", "[XXXXX813]", "[XXXXX814]", "[XXXXX815]", "[XXXXX816]", "[XXXXX817]", "[XXXXX818]", "[XXXXX819]", "[XXXXX820]", "[XXXXX821]", "[XXXXX822]", "[XXXXX823]", "[XXXXX824]", "[XXXXX825]", "[XXXXX826]", "[XXXXX827]", "[XXXXX828]", "[XXXXX829]", "[XXXXX830]", "[XXXXX831]", "[XXXXX832]", "[XXXXX833]", "[XXXXX834]", "[XXXXX835]", "[XXXXX836]", "[XXXXX837]", "[XXXXX838]", "[XXXXX839]", "[XXXXX840]", "[XXXXX841]", "[XXXXX842]", "[XXXXX843]", "[XXXXX844]", "[XXXXX845]", "[XXXXX846]", "[XXXXX847]", "[XXXXX848]", "[XXXXX849]", "[XXXXX850]", "[XXXXX851]", "[XXXXX852]", "[XXXXX853]", "[XXXXX854]", "[XXXXX855]", "[XXXXX856]", "[XXXXX857]", "[XXXXX858]", "[XXXXX859]", "[XXXXX860]", "[XXXXX861]", "[XXXXX862]", "[XXXXX863]", "[XXXXX864]", "[XXXXX865]", "[XXXXX866]", "[XXXXX867]", "[XXXXX868]", "[XXXXX869]", "[XXXXX870]", "[XXXXX871]", "[XXXXX872]", "[XXXXX873]"]}

spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3dc3adb7a0b88dcde1e9826ff411e8db9a35ee0ee3084a48603348b3459046a8
+size 936560

tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": false, "remove_space": true, "keep_accents": true, "bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "<unk>", "sep_token": "[SEP]", "pad_token": "<pad>", "cls_token": "[CLS]", "mask_token": {"content": "[MASK]", "single_word": false, "lstrip": true, "rstrip": false, "normalized": false, "__type": "AddedToken"}, "sp_model_kwargs": {}, "name_or_path": "models/5mb/bgp_latn_5mb", "model_input_names": ["input_ids", "attention_mask"], "special_tokens_map_file": "models/5mb/bgp_latn_5mb/special_tokens_map.json", "tokenizer_class": "AlbertTokenizer"}