Hugging Face's logo

Kowshik24
/
BanglaLM

Text Generation
Model card Files Community
Kowshik24 commited on
Commit
b20bc7d
•
1 Parent(s): c086ed4

tokenikzer added

Browse files
Files changed (1) hide show
  1. tokenizer_mappings.json +456 -0
tokenizer_mappings.json ADDED
@@ -0,0 +1,456 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "stoi": {
3
+ "\n": 0,
4
+ " ": 1,
5
+ "!": 2,
6
+ "\"": 3,
7
+ "#": 4,
8
+ "$": 5,
9
+ "%": 6,
10
+ "'": 7,
11
+ "(": 8,
12
+ ")": 9,
13
+ "*": 10,
14
+ "+": 11,
15
+ ",": 12,
16
+ "-": 13,
17
+ ".": 14,
18
+ "/": 15,
19
+ "0": 16,
20
+ "1": 17,
21
+ "2": 18,
22
+ "3": 19,
23
+ "4": 20,
24
+ "5": 21,
25
+ "6": 22,
26
+ "7": 23,
27
+ "8": 24,
28
+ "9": 25,
29
+ ":": 26,
30
+ ";": 27,
31
+ "=": 28,
32
+ ">": 29,
33
+ "?": 30,
34
+ "@": 31,
35
+ "A": 32,
36
+ "B": 33,
37
+ "C": 34,
38
+ "D": 35,
39
+ "E": 36,
40
+ "F": 37,
41
+ "G": 38,
42
+ "H": 39,
43
+ "I": 40,
44
+ "J": 41,
45
+ "K": 42,
46
+ "L": 43,
47
+ "M": 44,
48
+ "N": 45,
49
+ "O": 46,
50
+ "P": 47,
51
+ "Q": 48,
52
+ "R": 49,
53
+ "S": 50,
54
+ "T": 51,
55
+ "U": 52,
56
+ "V": 53,
57
+ "W": 54,
58
+ "X": 55,
59
+ "Y": 56,
60
+ "Z": 57,
61
+ "[": 58,
62
+ "]": 59,
63
+ "_": 60,
64
+ "`": 61,
65
+ "a": 62,
66
+ "b": 63,
67
+ "c": 64,
68
+ "d": 65,
69
+ "e": 66,
70
+ "f": 67,
71
+ "g": 68,
72
+ "h": 69,
73
+ "i": 70,
74
+ "j": 71,
75
+ "k": 72,
76
+ "l": 73,
77
+ "m": 74,
78
+ "n": 75,
79
+ "o": 76,
80
+ "p": 77,
81
+ "q": 78,
82
+ "r": 79,
83
+ "s": 80,
84
+ "t": 81,
85
+ "u": 82,
86
+ "v": 83,
87
+ "w": 84,
88
+ "x": 85,
89
+ "y": 86,
90
+ "z": 87,
91
+ "{": 88,
92
+ "|": 89,
93
+ "}": 90,
94
+ "~": 91,
95
+ "”": 92,
96
+ "£": 93,
97
+ "­": 94,
98
+ "°": 95,
99
+ "²": 96,
100
+ "´": 97,
101
+ "·": 98,
102
+ "é": 99,
103
+ "ï": 100,
104
+ "М": 101,
105
+ "П": 102,
106
+ "а": 103,
107
+ "б": 104,
108
+ "в": 105,
109
+ "г": 106,
110
+ "е": 107,
111
+ "ж": 108,
112
+ "з": 109,
113
+ "и": 110,
114
+ "й": 111,
115
+ "к": 112,
116
+ "л": 113,
117
+ "м": 114,
118
+ "н": 115,
119
+ "о": 116,
120
+ "п": 117,
121
+ "р": 118,
122
+ "с": 119,
123
+ "т": 120,
124
+ "у": 121,
125
+ "ф": 122,
126
+ "х": 123,
127
+ "ч": 124,
128
+ "ш": 125,
129
+ "ы": 126,
130
+ "ь": 127,
131
+ "ю": 128,
132
+ "я": 129,
133
+ "।": 130,
134
+ "ঁ": 131,
135
+ "ং": 132,
136
+ "ঃ": 133,
137
+ "অ": 134,
138
+ "আ": 135,
139
+ "ই": 136,
140
+ "ঈ": 137,
141
+ "উ": 138,
142
+ "ঊ": 139,
143
+ "ঋ": 140,
144
+ "এ": 141,
145
+ "ঐ": 142,
146
+ "ও": 143,
147
+ "ঔ": 144,
148
+ "ক": 145,
149
+ "খ": 146,
150
+ "গ": 147,
151
+ "ঘ": 148,
152
+ "ঙ": 149,
153
+ "চ": 150,
154
+ "ছ": 151,
155
+ "জ": 152,
156
+ "ঝ": 153,
157
+ "ঞ": 154,
158
+ "ট": 155,
159
+ "ঠ": 156,
160
+ "ড": 157,
161
+ "ঢ": 158,
162
+ "ণ": 159,
163
+ "ত": 160,
164
+ "থ": 161,
165
+ "দ": 162,
166
+ "ধ": 163,
167
+ "ন": 164,
168
+ "প": 165,
169
+ "ফ": 166,
170
+ "ব": 167,
171
+ "ভ": 168,
172
+ "ম": 169,
173
+ "য": 170,
174
+ "র": 171,
175
+ "ল": 172,
176
+ "শ": 173,
177
+ "ষ": 174,
178
+ "স": 175,
179
+ "হ": 176,
180
+ "়": 177,
181
+ "া": 178,
182
+ "ি": 179,
183
+ "ী": 180,
184
+ "ু": 181,
185
+ "ূ": 182,
186
+ "ৃ": 183,
187
+ "ে": 184,
188
+ "ৈ": 185,
189
+ "ো": 186,
190
+ "ৌ": 187,
191
+ "্": 188,
192
+ "ৎ": 189,
193
+ "ৗ": 190,
194
+ "ড়": 191,
195
+ "য়": 192,
196
+ "ৠ": 193,
197
+ "০": 194,
198
+ "১": 195,
199
+ "২": 196,
200
+ "৩": 197,
201
+ "৪": 198,
202
+ "৫": 199,
203
+ "৬": 200,
204
+ "৭": 201,
205
+ "৮": 202,
206
+ "৯": 203,
207
+ "ৰ": 204,
208
+ "ৱ": 205,
209
+ "৷": 206,
210
+ "৻": 207,
211
+ "‌": 208,
212
+ "‍": 209,
213
+ "‏": 210,
214
+ "–": 211,
215
+ "—": 212,
216
+ "‘": 213,
217
+ "’": 214,
218
+ "“": 215,
219
+ "”": 216,
220
+ "•": 217,
221
+ "…": 218,
222
+ "": 219,
223
+ "😂": 220,
224
+ "😅": 221,
225
+ "😜": 222,
226
+ "😡": 223,
227
+ "😰": 224
228
+ },
229
+ "itos": {
230
+ "0": "\n",
231
+ "1": " ",
232
+ "2": "!",
233
+ "3": "\"",
234
+ "4": "#",
235
+ "5": "$",
236
+ "6": "%",
237
+ "7": "'",
238
+ "8": "(",
239
+ "9": ")",
240
+ "10": "*",
241
+ "11": "+",
242
+ "12": ",",
243
+ "13": "-",
244
+ "14": ".",
245
+ "15": "/",
246
+ "16": "0",
247
+ "17": "1",
248
+ "18": "2",
249
+ "19": "3",
250
+ "20": "4",
251
+ "21": "5",
252
+ "22": "6",
253
+ "23": "7",
254
+ "24": "8",
255
+ "25": "9",
256
+ "26": ":",
257
+ "27": ";",
258
+ "28": "=",
259
+ "29": ">",
260
+ "30": "?",
261
+ "31": "@",
262
+ "32": "A",
263
+ "33": "B",
264
+ "34": "C",
265
+ "35": "D",
266
+ "36": "E",
267
+ "37": "F",
268
+ "38": "G",
269
+ "39": "H",
270
+ "40": "I",
271
+ "41": "J",
272
+ "42": "K",
273
+ "43": "L",
274
+ "44": "M",
275
+ "45": "N",
276
+ "46": "O",
277
+ "47": "P",
278
+ "48": "Q",
279
+ "49": "R",
280
+ "50": "S",
281
+ "51": "T",
282
+ "52": "U",
283
+ "53": "V",
284
+ "54": "W",
285
+ "55": "X",
286
+ "56": "Y",
287
+ "57": "Z",
288
+ "58": "[",
289
+ "59": "]",
290
+ "60": "_",
291
+ "61": "`",
292
+ "62": "a",
293
+ "63": "b",
294
+ "64": "c",
295
+ "65": "d",
296
+ "66": "e",
297
+ "67": "f",
298
+ "68": "g",
299
+ "69": "h",
300
+ "70": "i",
301
+ "71": "j",
302
+ "72": "k",
303
+ "73": "l",
304
+ "74": "m",
305
+ "75": "n",
306
+ "76": "o",
307
+ "77": "p",
308
+ "78": "q",
309
+ "79": "r",
310
+ "80": "s",
311
+ "81": "t",
312
+ "82": "u",
313
+ "83": "v",
314
+ "84": "w",
315
+ "85": "x",
316
+ "86": "y",
317
+ "87": "z",
318
+ "88": "{",
319
+ "89": "|",
320
+ "90": "}",
321
+ "91": "~",
322
+ "92": "”",
323
+ "93": "£",
324
+ "94": "­",
325
+ "95": "°",
326
+ "96": "²",
327
+ "97": "´",
328
+ "98": "·",
329
+ "99": "é",
330
+ "100": "ï",
331
+ "101": "М",
332
+ "102": "П",
333
+ "103": "а",
334
+ "104": "б",
335
+ "105": "в",
336
+ "106": "г",
337
+ "107": "е",
338
+ "108": "ж",
339
+ "109": "з",
340
+ "110": "и",
341
+ "111": "й",
342
+ "112": "к",
343
+ "113": "л",
344
+ "114": "м",
345
+ "115": "н",
346
+ "116": "о",
347
+ "117": "п",
348
+ "118": "р",
349
+ "119": "с",
350
+ "120": "т",
351
+ "121": "у",
352
+ "122": "ф",
353
+ "123": "х",
354
+ "124": "ч",
355
+ "125": "ш",
356
+ "126": "ы",
357
+ "127": "ь",
358
+ "128": "ю",
359
+ "129": "я",
360
+ "130": "।",
361
+ "131": "ঁ",
362
+ "132": "ং",
363
+ "133": "ঃ",
364
+ "134": "অ",
365
+ "135": "আ",
366
+ "136": "ই",
367
+ "137": "ঈ",
368
+ "138": "উ",
369
+ "139": "ঊ",
370
+ "140": "ঋ",
371
+ "141": "এ",
372
+ "142": "ঐ",
373
+ "143": "ও",
374
+ "144": "ঔ",
375
+ "145": "ক",
376
+ "146": "খ",
377
+ "147": "গ",
378
+ "148": "ঘ",
379
+ "149": "ঙ",
380
+ "150": "চ",
381
+ "151": "ছ",
382
+ "152": "জ",
383
+ "153": "ঝ",
384
+ "154": "ঞ",
385
+ "155": "ট",
386
+ "156": "ঠ",
387
+ "157": "ড",
388
+ "158": "ঢ",
389
+ "159": "ণ",
390
+ "160": "ত",
391
+ "161": "থ",
392
+ "162": "দ",
393
+ "163": "ধ",
394
+ "164": "ন",
395
+ "165": "প",
396
+ "166": "ফ",
397
+ "167": "ব",
398
+ "168": "ভ",
399
+ "169": "ম",
400
+ "170": "য",
401
+ "171": "র",
402
+ "172": "ল",
403
+ "173": "শ",
404
+ "174": "ষ",
405
+ "175": "স",
406
+ "176": "হ",
407
+ "177": "়",
408
+ "178": "া",
409
+ "179": "ি",
410
+ "180": "ী",
411
+ "181": "ু",
412
+ "182": "ূ",
413
+ "183": "ৃ",
414
+ "184": "ে",
415
+ "185": "ৈ",
416
+ "186": "ো",
417
+ "187": "ৌ",
418
+ "188": "্",
419
+ "189": "ৎ",
420
+ "190": "ৗ",
421
+ "191": "ড়",
422
+ "192": "য়",
423
+ "193": "ৠ",
424
+ "194": "০",
425
+ "195": "১",
426
+ "196": "২",
427
+ "197": "৩",
428
+ "198": "৪",
429
+ "199": "৫",
430
+ "200": "৬",
431
+ "201": "৭",
432
+ "202": "৮",
433
+ "203": "৯",
434
+ "204": "ৰ",
435
+ "205": "ৱ",
436
+ "206": "৷",
437
+ "207": "৻",
438
+ "208": "‌",
439
+ "209": "‍",
440
+ "210": "‏",
441
+ "211": "–",
442
+ "212": "—",
443
+ "213": "‘",
444
+ "214": "’",
445
+ "215": "“",
446
+ "216": "”",
447
+ "217": "•",
448
+ "218": "…",
449
+ "219": "",
450
+ "220": "😂",
451
+ "221": "😅",
452
+ "222": "😜",
453
+ "223": "😡",
454
+ "224": "😰"
455
+ }
456
+ }