{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "[UNK]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "[PAD]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "[CLS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "[SEP]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 4, "content": "[MASK]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": { "type": "Strip", "strip_left": true, "strip_right": true }, "pre_tokenizer": { "type": "WhitespaceSplit" }, "post_processor": { "type": "TemplateProcessing", "single": [ { "Sequence": { "id": "A", "type_id": 0 } }, { "SpecialToken": { "id": "[SEP]", "type_id": 0 } } ], "pair": [ { "Sequence": { "id": "A", "type_id": 0 } }, { "SpecialToken": { "id": "[SEP]", "type_id": 0 } }, { "Sequence": { "id": "B", "type_id": 1 } }, { "SpecialToken": { "id": "[SEP]", "type_id": 1 } } ], "special_tokens": { "[SEP]": { "id": "[SEP]", "ids": [ 3 ], "tokens": [ "[SEP]" ] } } }, "decoder": null, "model": { "type": "WordLevel", "vocab": { "[UNK]": 0, "[PAD]": 1, "[CLS]": 2, "[SEP]": 3, "[MASK]": 4, "69": 5, "79": 6, "98": 7, "26": 8, "29": 9, "65": 10, "86": 11, "91": 12, "92": 13, "05": 14, "67": 15, "85": 16, "03": 17, "11": 18, "21": 19, "28": 20, "45": 21, "48": 22, "50": 23, "52": 24, "57": 25, "59": 26, "63": 27, "66": 28, "76": 29, "87": 30, "02": 31, "06": 32, "17": 33, "25": 34, "35": 35, "44": 36, "51": 37, "53": 38, "55": 39, "56": 40, "62": 41, "64": 42, "71": 43, "82": 44, "88": 45, "94": 46, "95": 47, "96": 48, "07": 49, "10": 50, "12": 51, "14": 52, "18": 53, "19": 54, "20": 55, "24": 56, "41": 57, "43": 58, "49": 59, "58": 60, "61": 61, "72": 62, "74": 63, "81": 64, "90": 65, "00": 66, "08": 67, "09": 68, "23": 69, "33": 70, "34": 71, "36": 72, "39": 73, "68": 74, "97": 75, "99": 76, "13": 77, "15": 78, "22": 79, "27": 80, "30": 81, "38": 82, "47": 83, "75": 84, "77": 85, "80": 86, "93": 87, "31": 88, "32": 89, "37": 90, "40": 91, "42": 92, "46": 93, "54": 94, "01": 95, "04": 96, "16": 97, "60": 98, "70": 99, "73": 100, "78": 101, "83": 102, "89": 103 }, "unk_token": "[UNK]" } }