Commit
•
9516907
1
Parent(s):
e72d6c0
Upload tokenizer
Browse files- tokenizer.json +66 -2
tokenizer.json
CHANGED
|
@@ -1162,7 +1162,39 @@
|
|
| 1162 |
"tzaqe": 1100,
|
| 1163 |
"Ġpaagr": 1101,
|
| 1164 |
"ĠDyos": 1102,
|
| 1165 |
-
"choch": 1103
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1166 |
},
|
| 1167 |
"merges": [
|
| 1168 |
"t a",
|
|
@@ -2189,7 +2221,39 @@
|
|
| 2189 |
"tzaq e",
|
| 2190 |
"Ġpaa gr",
|
| 2191 |
"ĠDy os",
|
| 2192 |
-
"ch och"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2193 |
]
|
| 2194 |
}
|
| 2195 |
}
|
|
|
|
| 1162 |
"tzaqe": 1100,
|
| 1163 |
"Ġpaagr": 1101,
|
| 1164 |
"ĠDyos": 1102,
|
| 1165 |
+
"choch": 1103,
|
| 1166 |
+
"Ġtaj": 1104,
|
| 1167 |
+
"Ġxuq": 1105,
|
| 1168 |
+
"Ġqax": 1106,
|
| 1169 |
+
"ĠIs": 1107,
|
| 1170 |
+
"Ġtikoj": 1108,
|
| 1171 |
+
"rofesor": 1109,
|
| 1172 |
+
"Ġjkompayr": 1110,
|
| 1173 |
+
"jil": 1111,
|
| 1174 |
+
"aji": 1112,
|
| 1175 |
+
"Ġmaas": 1113,
|
| 1176 |
+
"Ġtrik": 1114,
|
| 1177 |
+
"ĠIn": 1115,
|
| 1178 |
+
"tiw": 1116,
|
| 1179 |
+
"Ġdyoos": 1117,
|
| 1180 |
+
"tami": 1118,
|
| 1181 |
+
"Ġretzel": 1119,
|
| 1182 |
+
"ekik": 1120,
|
| 1183 |
+
"stor": 1121,
|
| 1184 |
+
"Ġnada": 1122,
|
| 1185 |
+
"Ġtiqab": 1123,
|
| 1186 |
+
"Ġjujun": 1124,
|
| 1187 |
+
"wr": 1125,
|
| 1188 |
+
"Ġrel": 1126,
|
| 1189 |
+
"Ġpadre": 1127,
|
| 1190 |
+
"ara": 1128,
|
| 1191 |
+
"Ġuk": 1129,
|
| 1192 |
+
"Ġjy": 1130,
|
| 1193 |
+
"Ġrichooch": 1131,
|
| 1194 |
+
"Ġxkwin": 1132,
|
| 1195 |
+
"Ġtintij": 1133,
|
| 1196 |
+
"Ġant": 1134,
|
| 1197 |
+
"ĠAntons": 1135
|
| 1198 |
},
|
| 1199 |
"merges": [
|
| 1200 |
"t a",
|
|
|
|
| 2221 |
"tzaq e",
|
| 2222 |
"Ġpaa gr",
|
| 2223 |
"ĠDy os",
|
| 2224 |
+
"ch och",
|
| 2225 |
+
"Ġta j",
|
| 2226 |
+
"Ġx uq",
|
| 2227 |
+
"Ġqa x",
|
| 2228 |
+
"ĠI s",
|
| 2229 |
+
"Ġtik oj",
|
| 2230 |
+
"ro fesor",
|
| 2231 |
+
"Ġjkom payr",
|
| 2232 |
+
"j il",
|
| 2233 |
+
"aj i",
|
| 2234 |
+
"Ġm aas",
|
| 2235 |
+
"Ġtr ik",
|
| 2236 |
+
"ĠI n",
|
| 2237 |
+
"ti w",
|
| 2238 |
+
"Ġdy oos",
|
| 2239 |
+
"tam i",
|
| 2240 |
+
"Ġre tzel",
|
| 2241 |
+
"ek ik",
|
| 2242 |
+
"st or",
|
| 2243 |
+
"Ġnad a",
|
| 2244 |
+
"Ġtiqa b",
|
| 2245 |
+
"Ġjuj un",
|
| 2246 |
+
"w r",
|
| 2247 |
+
"Ġre l",
|
| 2248 |
+
"Ġp adre",
|
| 2249 |
+
"ar a",
|
| 2250 |
+
"Ġu k",
|
| 2251 |
+
"Ġj y",
|
| 2252 |
+
"Ġri chooch",
|
| 2253 |
+
"Ġxk win",
|
| 2254 |
+
"Ġtin tij",
|
| 2255 |
+
"Ġan t",
|
| 2256 |
+
"ĠAn tons"
|
| 2257 |
]
|
| 2258 |
}
|
| 2259 |
}
|