ronanki commited on
Commit
5058dba
1 Parent(s): 6c127aa

Upload tokenizer

Browse files
Files changed (4) hide show
  1. special_tokens_map.json +9 -0
  2. tokenizer.json +753 -0
  3. tokenizer_config.json +16 -0
  4. vocab.txt +600 -0
special_tokens_map.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "[CLS]",
3
+ "cls_token": "[CLS]",
4
+ "eos_token": "[SEP]",
5
+ "mask_token": "[MASK]",
6
+ "pad_token": "[PAD]",
7
+ "sep_token": "[SEP]",
8
+ "unk_token": "[UNK]"
9
+ }
tokenizer.json ADDED
@@ -0,0 +1,753 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "1.0",
3
+ "truncation": null,
4
+ "padding": null,
5
+ "added_tokens": [
6
+ {
7
+ "id": 0,
8
+ "content": "[PAD]",
9
+ "single_word": false,
10
+ "lstrip": false,
11
+ "rstrip": false,
12
+ "normalized": false,
13
+ "special": true
14
+ },
15
+ {
16
+ "id": 1,
17
+ "content": "[CLS]",
18
+ "single_word": false,
19
+ "lstrip": false,
20
+ "rstrip": false,
21
+ "normalized": false,
22
+ "special": true
23
+ },
24
+ {
25
+ "id": 2,
26
+ "content": "[SEP]",
27
+ "single_word": false,
28
+ "lstrip": false,
29
+ "rstrip": false,
30
+ "normalized": false,
31
+ "special": true
32
+ },
33
+ {
34
+ "id": 3,
35
+ "content": "[UNK]",
36
+ "single_word": false,
37
+ "lstrip": false,
38
+ "rstrip": false,
39
+ "normalized": false,
40
+ "special": true
41
+ },
42
+ {
43
+ "id": 4,
44
+ "content": "[MASK]",
45
+ "single_word": false,
46
+ "lstrip": false,
47
+ "rstrip": false,
48
+ "normalized": false,
49
+ "special": true
50
+ }
51
+ ],
52
+ "normalizer": {
53
+ "type": "Sequence",
54
+ "normalizers": [
55
+ {
56
+ "type": "NFD"
57
+ },
58
+ {
59
+ "type": "Lowercase"
60
+ },
61
+ {
62
+ "type": "StripAccents"
63
+ }
64
+ ]
65
+ },
66
+ "pre_tokenizer": {
67
+ "type": "Whitespace"
68
+ },
69
+ "post_processor": {
70
+ "type": "TemplateProcessing",
71
+ "single": [
72
+ {
73
+ "SpecialToken": {
74
+ "id": "[CLS]",
75
+ "type_id": 0
76
+ }
77
+ },
78
+ {
79
+ "Sequence": {
80
+ "id": "A",
81
+ "type_id": 0
82
+ }
83
+ },
84
+ {
85
+ "SpecialToken": {
86
+ "id": "[SEP]",
87
+ "type_id": 0
88
+ }
89
+ }
90
+ ],
91
+ "pair": [
92
+ {
93
+ "SpecialToken": {
94
+ "id": "[CLS]",
95
+ "type_id": 0
96
+ }
97
+ },
98
+ {
99
+ "Sequence": {
100
+ "id": "A",
101
+ "type_id": 0
102
+ }
103
+ },
104
+ {
105
+ "SpecialToken": {
106
+ "id": "[SEP]",
107
+ "type_id": 0
108
+ }
109
+ },
110
+ {
111
+ "Sequence": {
112
+ "id": "B",
113
+ "type_id": 1
114
+ }
115
+ },
116
+ {
117
+ "SpecialToken": {
118
+ "id": "[SEP]",
119
+ "type_id": 1
120
+ }
121
+ }
122
+ ],
123
+ "special_tokens": {
124
+ "[CLS]": {
125
+ "id": "[CLS]",
126
+ "ids": [
127
+ 1
128
+ ],
129
+ "tokens": [
130
+ "[CLS]"
131
+ ]
132
+ },
133
+ "[SEP]": {
134
+ "id": "[SEP]",
135
+ "ids": [
136
+ 2
137
+ ],
138
+ "tokens": [
139
+ "[SEP]"
140
+ ]
141
+ }
142
+ }
143
+ },
144
+ "decoder": null,
145
+ "model": {
146
+ "type": "WordPiece",
147
+ "unk_token": "[UNK]",
148
+ "continuing_subword_prefix": "##",
149
+ "max_input_chars_per_word": 100,
150
+ "vocab": {
151
+ "[PAD]": 0,
152
+ "[CLS]": 1,
153
+ "[SEP]": 2,
154
+ "[UNK]": 3,
155
+ "[MASK]": 4,
156
+ "\"": 5,
157
+ "%": 6,
158
+ "&": 7,
159
+ "'": 8,
160
+ "(": 9,
161
+ ")": 10,
162
+ "*": 11,
163
+ "+": 12,
164
+ ",": 13,
165
+ "-": 14,
166
+ ".": 15,
167
+ "/": 16,
168
+ "0": 17,
169
+ "1": 18,
170
+ "2": 19,
171
+ "3": 20,
172
+ "4": 21,
173
+ "5": 22,
174
+ "6": 23,
175
+ "7": 24,
176
+ "8": 25,
177
+ "9": 26,
178
+ ":": 27,
179
+ ";": 28,
180
+ "<": 29,
181
+ "=": 30,
182
+ ">": 31,
183
+ "[": 32,
184
+ "]": 33,
185
+ "_": 34,
186
+ "a": 35,
187
+ "b": 36,
188
+ "c": 37,
189
+ "d": 38,
190
+ "e": 39,
191
+ "f": 40,
192
+ "g": 41,
193
+ "h": 42,
194
+ "i": 43,
195
+ "j": 44,
196
+ "k": 45,
197
+ "l": 46,
198
+ "m": 47,
199
+ "n": 48,
200
+ "o": 49,
201
+ "p": 50,
202
+ "q": 51,
203
+ "r": 52,
204
+ "s": 53,
205
+ "t": 54,
206
+ "u": 55,
207
+ "v": 56,
208
+ "w": 57,
209
+ "x": 58,
210
+ "y": 59,
211
+ "z": 60,
212
+ "~": 61,
213
+ "®": 62,
214
+ "°": 63,
215
+ "²": 64,
216
+ "³": 65,
217
+ "µ": 66,
218
+ "ø": 67,
219
+ "α": 68,
220
+ "–": 69,
221
+ "##u": 70,
222
+ "##l": 71,
223
+ "##i": 72,
224
+ "##p": 73,
225
+ "##w": 74,
226
+ "##o": 75,
227
+ "##d": 76,
228
+ "##e": 77,
229
+ "##s": 78,
230
+ "##g": 79,
231
+ "##n": 80,
232
+ "##t": 81,
233
+ "##c": 82,
234
+ "##m": 83,
235
+ "##h": 84,
236
+ "##r": 85,
237
+ "##v": 86,
238
+ "##0": 87,
239
+ "##a": 88,
240
+ "##y": 89,
241
+ "##7": 90,
242
+ "##2": 91,
243
+ "##1": 92,
244
+ "##4": 93,
245
+ "##b": 94,
246
+ "##x": 95,
247
+ "##6": 96,
248
+ "##3": 97,
249
+ "##5": 98,
250
+ "##f": 99,
251
+ "##j": 100,
252
+ "##k": 101,
253
+ "##;": 102,
254
+ "##z": 103,
255
+ "##µ": 104,
256
+ "##8": 105,
257
+ "##9": 106,
258
+ "##%": 107,
259
+ "##(": 108,
260
+ "##q": 109,
261
+ "##]": 110,
262
+ "##)": 111,
263
+ "##,": 112,
264
+ "##_": 113,
265
+ "##\"": 114,
266
+ "##>": 115,
267
+ "##/": 116,
268
+ "##ø": 117,
269
+ "##<": 118,
270
+ "##=": 119,
271
+ "##-": 120,
272
+ "##er": 121,
273
+ "##in": 122,
274
+ "##on": 123,
275
+ "##at": 124,
276
+ "##ed": 125,
277
+ "##ro": 126,
278
+ "##en": 127,
279
+ "##an": 128,
280
+ "##ic": 129,
281
+ "##as": 130,
282
+ "##ar": 131,
283
+ "##or": 132,
284
+ "##al": 133,
285
+ "##ct": 134,
286
+ "##it": 135,
287
+ "##ol": 136,
288
+ "##il": 137,
289
+ "##ion": 138,
290
+ "##ing": 139,
291
+ "##es": 140,
292
+ "##et": 141,
293
+ "##el": 142,
294
+ "##id": 143,
295
+ "in": 144,
296
+ "##le": 145,
297
+ "tr": 146,
298
+ "##um": 147,
299
+ "##yl": 148,
300
+ "en": 149,
301
+ "##ent": 150,
302
+ "##te": 151,
303
+ "##re": 152,
304
+ "##ation": 153,
305
+ "co": 154,
306
+ "##st": 155,
307
+ "##od": 156,
308
+ "pl": 157,
309
+ "##am": 158,
310
+ "##rom": 159,
311
+ "con": 160,
312
+ "##ity": 161,
313
+ "##80": 162,
314
+ "##ric": 163,
315
+ "##ul": 164,
316
+ "di": 165,
317
+ "from": 166,
318
+ "##ene": 167,
319
+ "##ig": 168,
320
+ "##580": 169,
321
+ "##5804": 170,
322
+ "pro": 171,
323
+ "ele": 172,
324
+ "##ut": 173,
325
+ "##ck": 174,
326
+ "re": 175,
327
+ "elect": 176,
328
+ "##lo": 177,
329
+ "##ur": 178,
330
+ "##ate": 179,
331
+ "a1": 180,
332
+ "a3": 181,
333
+ "##ad": 182,
334
+ "al": 183,
335
+ "##ix": 184,
336
+ "##ated": 185,
337
+ "pa": 186,
338
+ "##igh": 187,
339
+ "##ine": 188,
340
+ "##du": 189,
341
+ "##oly": 190,
342
+ "mix": 191,
343
+ "##ac": 192,
344
+ "car": 193,
345
+ "ste": 194,
346
+ "at": 195,
347
+ "##ood": 196,
348
+ "st": 197,
349
+ "electric": 198,
350
+ "##ow": 199,
351
+ "##im": 200,
352
+ "##sp": 201,
353
+ "##em": 202,
354
+ "##ium": 203,
355
+ "20": 204,
356
+ "gr": 205,
357
+ "poly": 206,
358
+ "##ard": 207,
359
+ "##eth": 208,
360
+ "##ap": 209,
361
+ "##ill": 210,
362
+ "mm": 211,
363
+ "electricity": 212,
364
+ "##uck": 213,
365
+ "truck": 214,
366
+ "##ip": 215,
367
+ "##ber": 216,
368
+ "##ch": 217,
369
+ "##duct": 218,
370
+ "##15804": 219,
371
+ "en15804": 220,
372
+ "cont": 221,
373
+ "##pp": 222,
374
+ "##ort": 223,
375
+ "##ide": 224,
376
+ "##ight": 225,
377
+ "plan": 226,
378
+ "sh": 227,
379
+ "of": 228,
380
+ "##nd": 229,
381
+ "##ast": 230,
382
+ "product": 231,
383
+ "##oad": 232,
384
+ "##eat": 233,
385
+ "##ge": 234,
386
+ "##wood": 235,
387
+ "##ased": 236,
388
+ "pow": 237,
389
+ "steel": 238,
390
+ "##ver": 239,
391
+ "##ess": 240,
392
+ "ac": 241,
393
+ "##ater": 242,
394
+ "##00": 243,
395
+ "##and": 244,
396
+ "content": 245,
397
+ "par": 246,
398
+ "##ri": 247,
399
+ "com": 248,
400
+ "ro": 249,
401
+ "was": 250,
402
+ "##yload": 251,
403
+ "payload": 252,
404
+ "##th": 253,
405
+ "ch": 254,
406
+ "##ist": 255,
407
+ "##ter": 256,
408
+ "##of": 257,
409
+ "fi": 258,
410
+ "gas": 259,
411
+ "##ass": 260,
412
+ "waste": 261,
413
+ "cap": 262,
414
+ "##ylene": 263,
415
+ "gro": 264,
416
+ "##ith": 265,
417
+ "##ane": 266,
418
+ "power": 267,
419
+ "tran": 268,
420
+ "##cl": 269,
421
+ "15804": 270,
422
+ "app": 271,
423
+ "based": 272,
424
+ "##is": 273,
425
+ "capac": 274,
426
+ "##ure": 275,
427
+ "th": 276,
428
+ "and": 277,
429
+ "water": 278,
430
+ "##om": 279,
431
+ "plant": 280,
432
+ "for": 281,
433
+ "un": 282,
434
+ "##led": 283,
435
+ "grid": 284,
436
+ "##us": 285,
437
+ "##eet": 286,
438
+ "part": 287,
439
+ "##yd": 288,
440
+ "##2o": 289,
441
+ "##est": 290,
442
+ "##old": 291,
443
+ "by": 292,
444
+ "##co": 293,
445
+ "oil": 294,
446
+ "##eight": 295,
447
+ "##ce": 296,
448
+ "bo": 297,
449
+ "##rox": 298,
450
+ "res": 299,
451
+ "##gy": 300,
452
+ "##ss": 301,
453
+ "##aw": 302,
454
+ "sheet": 303,
455
+ "h2o": 304,
456
+ "us": 305,
457
+ "##ly": 306,
458
+ "with": 307,
459
+ "##esel": 308,
460
+ "transp": 309,
461
+ "capacity": 310,
462
+ "mo": 311,
463
+ "##iner": 312,
464
+ "transport": 313,
465
+ "##uro": 314,
466
+ "weight": 315,
467
+ "##iz": 316,
468
+ "),": 317,
469
+ "##bon": 318,
470
+ "bi": 319,
471
+ "on": 320,
472
+ "##ergy": 321,
473
+ "diesel": 322,
474
+ "energy": 323,
475
+ "euro": 324,
476
+ "##un": 325,
477
+ "##lu": 326,
478
+ "##rect": 327,
479
+ "##age": 328,
480
+ "approx": 329,
481
+ "se": 330,
482
+ "gross": 331,
483
+ "carbon": 332,
484
+ "cr": 333,
485
+ "##imation": 334,
486
+ "approximation": 335,
487
+ "##cy": 336,
488
+ "##ment": 337,
489
+ "##fill": 338,
490
+ "to": 339,
491
+ "12": 340,
492
+ "acid": 341,
493
+ "land": 342,
494
+ "##umin": 343,
495
+ "op": 344,
496
+ "##ug": 345,
497
+ "##av": 346,
498
+ "landfill": 347,
499
+ "##ial": 348,
500
+ "alumin": 349,
501
+ "##mm": 350,
502
+ "##put": 351,
503
+ "##ural": 352,
504
+ "allo": 353,
505
+ "##cess": 354,
506
+ "fiber": 355,
507
+ "eth": 356,
508
+ "powered": 357,
509
+ "process": 358,
510
+ "production": 359,
511
+ "ex": 360,
512
+ "vd": 361,
513
+ "##act": 362,
514
+ "moist": 363,
515
+ "alloc": 364,
516
+ "##iv": 365,
517
+ "##cr": 366,
518
+ "##ain": 367,
519
+ "##lor": 368,
520
+ "##ab": 369,
521
+ "de": 370,
522
+ "##mg": 371,
523
+ "hyd": 372,
524
+ "kw": 373,
525
+ "pr": 374,
526
+ "##ile": 375,
527
+ "##one": 376,
528
+ "moisture": 377,
529
+ "hard": 378,
530
+ "##amine": 379,
531
+ "rol": 380,
532
+ "open": 381,
533
+ "##os": 382,
534
+ "##all": 383,
535
+ "te": 384,
536
+ "##ne": 385,
537
+ "##iler": 386,
538
+ "##con": 387,
539
+ "##for": 388,
540
+ "kg": 389,
541
+ "##erm": 390,
542
+ "dri": 391,
543
+ "input": 392,
544
+ "comp": 393,
545
+ "cf": 394,
546
+ "so": 395,
547
+ "cem": 396,
548
+ "##ph": 397,
549
+ "treat": 398,
550
+ "treatment": 399,
551
+ "ag": 400,
552
+ "lc": 401,
553
+ "##der": 402,
554
+ "allocation": 403,
555
+ "rolled": 404,
556
+ "flo": 405,
557
+ "mun": 406,
558
+ "##icip": 407,
559
+ "##ctor": 408,
560
+ "plast": 409,
561
+ "roof": 410,
562
+ "municip": 411,
563
+ "municipal": 412,
564
+ "10": 413,
565
+ "cold": 414,
566
+ "##pe": 415,
567
+ "##ay": 416,
568
+ "##verage": 417,
569
+ "average": 418,
570
+ "##cycl": 419,
571
+ "recycl": 420,
572
+ "dried": 421,
573
+ "meth": 422,
574
+ "wood": 423,
575
+ "ep": 424,
576
+ "nat": 425,
577
+ "##br": 426,
578
+ "##og": 427,
579
+ "60": 428,
580
+ "##ance": 429,
581
+ "201": 430,
582
+ "natural": 431,
583
+ "steam": 432,
584
+ "aluminium": 433,
585
+ "##ite": 434,
586
+ "therm": 435,
587
+ "high": 436,
588
+ "wh": 437,
589
+ "##di": 438,
590
+ "##sion": 439,
591
+ "reg": 440,
592
+ "##ot": 441,
593
+ "14": 442,
594
+ "##uel": 443,
595
+ "##bo": 444,
596
+ "pan": 445,
597
+ "ply": 446,
598
+ "fuel": 447,
599
+ "hydro": 448,
600
+ "ha": 449,
601
+ "mod": 450,
602
+ "sol": 451,
603
+ "##ub": 452,
604
+ "##ust": 453,
605
+ "##lass": 454,
606
+ "##ir": 455,
607
+ "conne": 456,
608
+ "connector": 457,
609
+ "c3": 458,
610
+ "lum": 459,
611
+ "inc": 460,
612
+ "chlor": 461,
613
+ "region": 462,
614
+ "eg": 463,
615
+ "sof": 464,
616
+ "##sul": 465,
617
+ "##ry": 466,
618
+ "coated": 467,
619
+ "floor": 468,
620
+ "soft": 469,
621
+ "cor": 470,
622
+ "light": 471,
623
+ "comb": 472,
624
+ "board": 473,
625
+ "lumber": 474,
626
+ "c4": 475,
627
+ "##kv": 476,
628
+ "kil": 477,
629
+ "mel": 478,
630
+ "kiln": 479,
631
+ "bf": 480,
632
+ "coat": 481,
633
+ "a5": 482,
634
+ "recycling": 483,
635
+ "##x1": 484,
636
+ "##ical": 485,
637
+ "inciner": 486,
638
+ "##mer": 487,
639
+ "haul": 488,
640
+ "##ens": 489,
641
+ "##ode": 490,
642
+ "cut": 491,
643
+ "fl": 492,
644
+ "he": 493,
645
+ "pet": 494,
646
+ "up": 495,
647
+ "##anul": 496,
648
+ "direct": 497,
649
+ "granul": 498,
650
+ "incineration": 499,
651
+ "heat": 500,
652
+ "melamine": 501,
653
+ "but": 502,
654
+ "softwood": 503,
655
+ "rene": 504,
656
+ "insul": 505,
657
+ "200": 506,
658
+ "vda": 507,
659
+ "cement": 508,
660
+ "renew": 509,
661
+ "e1": 510,
662
+ "##ethane": 511,
663
+ "15": 512,
664
+ "pin": 513,
665
+ "##uf": 514,
666
+ "##dust": 515,
667
+ "25": 516,
668
+ "hd": 517,
669
+ "sm": 518,
670
+ "sy": 519,
671
+ "slu": 520,
672
+ "##ia": 521,
673
+ "##op": 522,
674
+ "##dge": 523,
675
+ "##yp": 524,
676
+ "##int": 525,
677
+ "##ingle": 526,
678
+ "dist": 527,
679
+ "roofing": 528,
680
+ "granulate": 529,
681
+ "sludge": 530,
682
+ "mill": 531,
683
+ "sp": 532,
684
+ "##ding": 533,
685
+ "electr": 534,
686
+ "##board": 535,
687
+ "##ue": 536,
688
+ "##if": 537,
689
+ "##ong": 538,
690
+ "flooring": 539,
691
+ "22": 540,
692
+ "fo": 541,
693
+ "pe": 542,
694
+ "##infor": 543,
695
+ "reinfor": 544,
696
+ "reinforc": 545,
697
+ "##ied": 546,
698
+ "##ox": 547,
699
+ "##tr": 548,
700
+ "mem": 549,
701
+ "##ive": 550,
702
+ "nm": 551,
703
+ "single": 552,
704
+ "##per": 553,
705
+ "##fin": 554,
706
+ "##icle": 555,
707
+ "incl": 556,
708
+ "##sph": 557,
709
+ "tex": 558,
710
+ "plywood": 559,
711
+ "model": 560,
712
+ "16": 561,
713
+ "18": 562,
714
+ "man": 563,
715
+ "ph": 564,
716
+ "saw": 565,
717
+ "##oline": 566,
718
+ "indi": 567,
719
+ "gasoline": 568,
720
+ "indirect": 569,
721
+ "dw": 570,
722
+ "##yr": 571,
723
+ "##ete": 572,
724
+ "##ell": 573,
725
+ "resist": 574,
726
+ "hardwood": 575,
727
+ "sil": 576,
728
+ "##anol": 577,
729
+ "ba": 578,
730
+ "coating": 579,
731
+ "000": 580,
732
+ "pip": 581,
733
+ "##ool": 582,
734
+ "boiler": 583,
735
+ "sul": 584,
736
+ "indust": 585,
737
+ "membr": 586,
738
+ "##ous": 587,
739
+ "##ides": 588,
740
+ "##crete": 589,
741
+ "membrane": 590,
742
+ "resin": 591,
743
+ "insulation": 592,
744
+ "32": 593,
745
+ "econ": 594,
746
+ "ic": 595,
747
+ "econom": 596,
748
+ "pv": 597,
749
+ "##pylene": 598,
750
+ "##hyd": 599
751
+ }
752
+ }
753
+ }
tokenizer_config.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": "[CLS]",
3
+ "cls_token": "[CLS]",
4
+ "do_lower_case": true,
5
+ "eos_token": "[SEP]",
6
+ "mask_token": "[MASK]",
7
+ "model_max_length": 1000000000000000019884624838656,
8
+ "name_or_path": "/Users/avinashronanki/PycharmProjects/machine_learning/tokenization_pipeline/new_tokenizer",
9
+ "pad_token": "[PAD]",
10
+ "sep_token": "[SEP]",
11
+ "special_tokens_map_file": "/Users/avinashronanki/PycharmProjects/machine_learning/tokenization_pipeline/new_tokenizer/special_tokens_map.json",
12
+ "strip_accents": null,
13
+ "tokenize_chinese_chars": true,
14
+ "tokenizer_class": "BertTokenizer",
15
+ "unk_token": "[UNK]"
16
+ }
vocab.txt ADDED
@@ -0,0 +1,600 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [PAD]
2
+ [CLS]
3
+ [SEP]
4
+ [UNK]
5
+ [MASK]
6
+ "
7
+ %
8
+ &
9
+ '
10
+ (
11
+ )
12
+ *
13
+ +
14
+ ,
15
+ -
16
+ .
17
+ /
18
+ 0
19
+ 1
20
+ 2
21
+ 3
22
+ 4
23
+ 5
24
+ 6
25
+ 7
26
+ 8
27
+ 9
28
+ :
29
+ ;
30
+ <
31
+ =
32
+ >
33
+ [
34
+ ]
35
+ _
36
+ a
37
+ b
38
+ c
39
+ d
40
+ e
41
+ f
42
+ g
43
+ h
44
+ i
45
+ j
46
+ k
47
+ l
48
+ m
49
+ n
50
+ o
51
+ p
52
+ q
53
+ r
54
+ s
55
+ t
56
+ u
57
+ v
58
+ w
59
+ x
60
+ y
61
+ z
62
+ ~
63
+ ®
64
+ °
65
+ ²
66
+ ³
67
+ µ
68
+ ø
69
+ α
70
+
71
+ ##u
72
+ ##l
73
+ ##i
74
+ ##p
75
+ ##w
76
+ ##o
77
+ ##d
78
+ ##e
79
+ ##s
80
+ ##g
81
+ ##n
82
+ ##t
83
+ ##c
84
+ ##m
85
+ ##h
86
+ ##r
87
+ ##v
88
+ ##0
89
+ ##a
90
+ ##y
91
+ ##7
92
+ ##2
93
+ ##1
94
+ ##4
95
+ ##b
96
+ ##x
97
+ ##6
98
+ ##3
99
+ ##5
100
+ ##f
101
+ ##j
102
+ ##k
103
+ ##;
104
+ ##z
105
+ ##µ
106
+ ##8
107
+ ##9
108
+ ##%
109
+ ##(
110
+ ##q
111
+ ##]
112
+ ##)
113
+ ##,
114
+ ##_
115
+ ##"
116
+ ##>
117
+ ##/
118
+ ##ø
119
+ ##<
120
+ ##=
121
+ ##-
122
+ ##er
123
+ ##in
124
+ ##on
125
+ ##at
126
+ ##ed
127
+ ##ro
128
+ ##en
129
+ ##an
130
+ ##ic
131
+ ##as
132
+ ##ar
133
+ ##or
134
+ ##al
135
+ ##ct
136
+ ##it
137
+ ##ol
138
+ ##il
139
+ ##ion
140
+ ##ing
141
+ ##es
142
+ ##et
143
+ ##el
144
+ ##id
145
+ in
146
+ ##le
147
+ tr
148
+ ##um
149
+ ##yl
150
+ en
151
+ ##ent
152
+ ##te
153
+ ##re
154
+ ##ation
155
+ co
156
+ ##st
157
+ ##od
158
+ pl
159
+ ##am
160
+ ##rom
161
+ con
162
+ ##ity
163
+ ##80
164
+ ##ric
165
+ ##ul
166
+ di
167
+ from
168
+ ##ene
169
+ ##ig
170
+ ##580
171
+ ##5804
172
+ pro
173
+ ele
174
+ ##ut
175
+ ##ck
176
+ re
177
+ elect
178
+ ##lo
179
+ ##ur
180
+ ##ate
181
+ a1
182
+ a3
183
+ ##ad
184
+ al
185
+ ##ix
186
+ ##ated
187
+ pa
188
+ ##igh
189
+ ##ine
190
+ ##du
191
+ ##oly
192
+ mix
193
+ ##ac
194
+ car
195
+ ste
196
+ at
197
+ ##ood
198
+ st
199
+ electric
200
+ ##ow
201
+ ##im
202
+ ##sp
203
+ ##em
204
+ ##ium
205
+ 20
206
+ gr
207
+ poly
208
+ ##ard
209
+ ##eth
210
+ ##ap
211
+ ##ill
212
+ mm
213
+ electricity
214
+ ##uck
215
+ truck
216
+ ##ip
217
+ ##ber
218
+ ##ch
219
+ ##duct
220
+ ##15804
221
+ en15804
222
+ cont
223
+ ##pp
224
+ ##ort
225
+ ##ide
226
+ ##ight
227
+ plan
228
+ sh
229
+ of
230
+ ##nd
231
+ ##ast
232
+ product
233
+ ##oad
234
+ ##eat
235
+ ##ge
236
+ ##wood
237
+ ##ased
238
+ pow
239
+ steel
240
+ ##ver
241
+ ##ess
242
+ ac
243
+ ##ater
244
+ ##00
245
+ ##and
246
+ content
247
+ par
248
+ ##ri
249
+ com
250
+ ro
251
+ was
252
+ ##yload
253
+ payload
254
+ ##th
255
+ ch
256
+ ##ist
257
+ ##ter
258
+ ##of
259
+ fi
260
+ gas
261
+ ##ass
262
+ waste
263
+ cap
264
+ ##ylene
265
+ gro
266
+ ##ith
267
+ ##ane
268
+ power
269
+ tran
270
+ ##cl
271
+ 15804
272
+ app
273
+ based
274
+ ##is
275
+ capac
276
+ ##ure
277
+ th
278
+ and
279
+ water
280
+ ##om
281
+ plant
282
+ for
283
+ un
284
+ ##led
285
+ grid
286
+ ##us
287
+ ##eet
288
+ part
289
+ ##yd
290
+ ##2o
291
+ ##est
292
+ ##old
293
+ by
294
+ ##co
295
+ oil
296
+ ##eight
297
+ ##ce
298
+ bo
299
+ ##rox
300
+ res
301
+ ##gy
302
+ ##ss
303
+ ##aw
304
+ sheet
305
+ h2o
306
+ us
307
+ ##ly
308
+ with
309
+ ##esel
310
+ transp
311
+ capacity
312
+ mo
313
+ ##iner
314
+ transport
315
+ ##uro
316
+ weight
317
+ ##iz
318
+ ),
319
+ ##bon
320
+ bi
321
+ on
322
+ ##ergy
323
+ diesel
324
+ energy
325
+ euro
326
+ ##un
327
+ ##lu
328
+ ##rect
329
+ ##age
330
+ approx
331
+ se
332
+ gross
333
+ carbon
334
+ cr
335
+ ##imation
336
+ approximation
337
+ ##cy
338
+ ##ment
339
+ ##fill
340
+ to
341
+ 12
342
+ acid
343
+ land
344
+ ##umin
345
+ op
346
+ ##ug
347
+ ##av
348
+ landfill
349
+ ##ial
350
+ alumin
351
+ ##mm
352
+ ##put
353
+ ##ural
354
+ allo
355
+ ##cess
356
+ fiber
357
+ eth
358
+ powered
359
+ process
360
+ production
361
+ ex
362
+ vd
363
+ ##act
364
+ moist
365
+ alloc
366
+ ##iv
367
+ ##cr
368
+ ##ain
369
+ ##lor
370
+ ##ab
371
+ de
372
+ ##mg
373
+ hyd
374
+ kw
375
+ pr
376
+ ##ile
377
+ ##one
378
+ moisture
379
+ hard
380
+ ##amine
381
+ rol
382
+ open
383
+ ##os
384
+ ##all
385
+ te
386
+ ##ne
387
+ ##iler
388
+ ##con
389
+ ##for
390
+ kg
391
+ ##erm
392
+ dri
393
+ input
394
+ comp
395
+ cf
396
+ so
397
+ cem
398
+ ##ph
399
+ treat
400
+ treatment
401
+ ag
402
+ lc
403
+ ##der
404
+ allocation
405
+ rolled
406
+ flo
407
+ mun
408
+ ##icip
409
+ ##ctor
410
+ plast
411
+ roof
412
+ municip
413
+ municipal
414
+ 10
415
+ cold
416
+ ##pe
417
+ ##ay
418
+ ##verage
419
+ average
420
+ ##cycl
421
+ recycl
422
+ dried
423
+ meth
424
+ wood
425
+ ep
426
+ nat
427
+ ##br
428
+ ##og
429
+ 60
430
+ ##ance
431
+ 201
432
+ natural
433
+ steam
434
+ aluminium
435
+ ##ite
436
+ therm
437
+ high
438
+ wh
439
+ ##di
440
+ ##sion
441
+ reg
442
+ ##ot
443
+ 14
444
+ ##uel
445
+ ##bo
446
+ pan
447
+ ply
448
+ fuel
449
+ hydro
450
+ ha
451
+ mod
452
+ sol
453
+ ##ub
454
+ ##ust
455
+ ##lass
456
+ ##ir
457
+ conne
458
+ connector
459
+ c3
460
+ lum
461
+ inc
462
+ chlor
463
+ region
464
+ eg
465
+ sof
466
+ ##sul
467
+ ##ry
468
+ coated
469
+ floor
470
+ soft
471
+ cor
472
+ light
473
+ comb
474
+ board
475
+ lumber
476
+ c4
477
+ ##kv
478
+ kil
479
+ mel
480
+ kiln
481
+ bf
482
+ coat
483
+ a5
484
+ recycling
485
+ ##x1
486
+ ##ical
487
+ inciner
488
+ ##mer
489
+ haul
490
+ ##ens
491
+ ##ode
492
+ cut
493
+ fl
494
+ he
495
+ pet
496
+ up
497
+ ##anul
498
+ direct
499
+ granul
500
+ incineration
501
+ heat
502
+ melamine
503
+ but
504
+ softwood
505
+ rene
506
+ insul
507
+ 200
508
+ vda
509
+ cement
510
+ renew
511
+ e1
512
+ ##ethane
513
+ 15
514
+ pin
515
+ ##uf
516
+ ##dust
517
+ 25
518
+ hd
519
+ sm
520
+ sy
521
+ slu
522
+ ##ia
523
+ ##op
524
+ ##dge
525
+ ##yp
526
+ ##int
527
+ ##ingle
528
+ dist
529
+ roofing
530
+ granulate
531
+ sludge
532
+ mill
533
+ sp
534
+ ##ding
535
+ electr
536
+ ##board
537
+ ##ue
538
+ ##if
539
+ ##ong
540
+ flooring
541
+ 22
542
+ fo
543
+ pe
544
+ ##infor
545
+ reinfor
546
+ reinforc
547
+ ##ied
548
+ ##ox
549
+ ##tr
550
+ mem
551
+ ##ive
552
+ nm
553
+ single
554
+ ##per
555
+ ##fin
556
+ ##icle
557
+ incl
558
+ ##sph
559
+ tex
560
+ plywood
561
+ model
562
+ 16
563
+ 18
564
+ man
565
+ ph
566
+ saw
567
+ ##oline
568
+ indi
569
+ gasoline
570
+ indirect
571
+ dw
572
+ ##yr
573
+ ##ete
574
+ ##ell
575
+ resist
576
+ hardwood
577
+ sil
578
+ ##anol
579
+ ba
580
+ coating
581
+ 000
582
+ pip
583
+ ##ool
584
+ boiler
585
+ sul
586
+ indust
587
+ membr
588
+ ##ous
589
+ ##ides
590
+ ##crete
591
+ membrane
592
+ resin
593
+ insulation
594
+ 32
595
+ econ
596
+ ic
597
+ econom
598
+ pv
599
+ ##pylene
600
+ ##hyd