{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": true }, "post_processor": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": false, "use_regex": true }, "decoder": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true }, "model": { "type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": "", "end_of_word_suffix": "", "fuse_unk": false, "vocab": { "<|endoftext|>": 0, "!": 1, "\"": 2, "#": 3, "$": 4, "%": 5, "&": 6, "'": 7, "(": 8, ")": 9, "*": 10, "+": 11, ",": 12, "-": 13, ".": 14, "/": 15, "0": 16, "1": 17, "2": 18, "3": 19, "4": 20, "5": 21, "6": 22, "7": 23, "8": 24, "9": 25, ":": 26, ";": 27, "<": 28, "=": 29, ">": 30, "?": 31, "@": 32, "A": 33, "B": 34, "C": 35, "D": 36, "E": 37, "F": 38, "G": 39, "H": 40, "I": 41, "J": 42, "K": 43, "L": 44, "M": 45, "N": 46, "O": 47, "P": 48, "Q": 49, "R": 50, "S": 51, "T": 52, "U": 53, "V": 54, "W": 55, "X": 56, "Y": 57, "Z": 58, "[": 59, "\\": 60, "]": 61, "^": 62, "_": 63, "`": 64, "a": 65, "b": 66, "c": 67, "d": 68, "e": 69, "f": 70, "g": 71, "h": 72, "i": 73, "j": 74, "k": 75, "l": 76, "m": 77, "n": 78, "o": 79, "p": 80, "q": 81, "r": 82, "s": 83, "t": 84, "u": 85, "v": 86, "w": 87, "x": 88, "y": 89, "z": 90, "{": 91, "|": 92, "}": 93, "~": 94, "¡": 95, "¢": 96, "£": 97, "¤": 98, "¥": 99, "¦": 100, "§": 101, "¨": 102, "©": 103, "ª": 104, "«": 105, "¬": 106, "®": 107, "¯": 108, "°": 109, "±": 110, "²": 111, "³": 112, "´": 113, "µ": 114, "¶": 115, "·": 116, "¸": 117, "¹": 118, "º": 119, "»": 120, "¼": 121, "½": 122, "¾": 123, "¿": 124, "À": 125, "Á": 126, "Â": 127, "Ã": 128, "Ä": 129, "Å": 130, "Æ": 131, "Ç": 132, "È": 133, "É": 134, "Ê": 135, "Ë": 136, "Ì": 137, "Í": 138, "Î": 139, "Ï": 140, "Ð": 141, "Ñ": 142, "Ò": 143, "Ó": 144, "Ô": 145, "Õ": 146, "Ö": 147, "×": 148, "Ø": 149, "Ù": 150, "Ú": 151, "Û": 152, "Ü": 153, "Ý": 154, "Þ": 155, "ß": 156, "à": 157, "á": 158, "â": 159, "ã": 160, "ä": 161, "å": 162, "æ": 163, "ç": 164, "è": 165, "é": 166, "ê": 167, "ë": 168, "ì": 169, "í": 170, "î": 171, "ï": 172, "ð": 173, "ñ": 174, "ò": 175, "ó": 176, "ô": 177, "õ": 178, "ö": 179, "÷": 180, "ø": 181, "ù": 182, "ú": 183, "û": 184, "ü": 185, "ý": 186, "þ": 187, "ÿ": 188, "Ā": 189, "ā": 190, "Ă": 191, "ă": 192, "Ą": 193, "ą": 194, "Ć": 195, "ć": 196, "Ĉ": 197, "ĉ": 198, "Ċ": 199, "ċ": 200, "Č": 201, "č": 202, "Ď": 203, "ď": 204, "Đ": 205, "đ": 206, "Ē": 207, "ē": 208, "Ĕ": 209, "ĕ": 210, "Ė": 211, "ė": 212, "Ę": 213, "ę": 214, "Ě": 215, "ě": 216, "Ĝ": 217, "ĝ": 218, "Ğ": 219, "ğ": 220, "Ġ": 221, "ġ": 222, "Ģ": 223, "ģ": 224, "Ĥ": 225, "ĥ": 226, "Ħ": 227, "ħ": 228, "Ĩ": 229, "ĩ": 230, "Ī": 231, "ī": 232, "Ĭ": 233, "ĭ": 234, "Į": 235, "į": 236, "İ": 237, "ı": 238, "IJ": 239, "ij": 240, "Ĵ": 241, "ĵ": 242, "Ķ": 243, "ķ": 244, "ĸ": 245, "Ĺ": 246, "ĺ": 247, "Ļ": 248, "ļ": 249, "Ľ": 250, "ľ": 251, "Ŀ": 252, "ŀ": 253, "Ł": 254, "ł": 255, "Ń": 256, "Ġ4": 257, "Ġ47": 258, "00": 259, "AN": 260, "ANN": 261, "AL": 262, "003": 263, "TI": 264, "Ġ48": 265, "PAL": 266, "007": 267, "03": 268, "LI": 269, "FLI": 270, "032": 271, "AB": 272, "TAL": 273, "002": 274, "FI": 275, "FIC": 276, "034": 277, "15": 278, "AH": 279, "AHH": 280, "02": 281, "2003": 282, "01": 283, "LAHH": 284, "SI": 285, "900": 286, "802": 287, "USI": 288, "70": 289, "870": 290, "151": 291, "PI": 292, "1512": 293, "47": 294, "AI": 295, "039": 296, "001": 297, "1503": 298, "50": 299, "850": 300, "XL": 301, "LIC": 302, "008": 303, "PALL": 304, "010": 305, "60": 306, "011": 307, "PD": 308, "18": 309, "ABC": 310, "XA": 311, "SIO": 312, "605": 313, "PDAH": 314, "ĠSIO": 315, "004": 316, "AAHH": 317, "021": 318, "150": 319, "CD": 320, "005": 321, "48": 322, "152": 323, "17": 324, "019": 325, "FS": 326, "FSL": 327, "PIC": 328, "04": 329, "153": 330, "608": 331, "1535": 332, "015": 333, "013": 334, "1505": 335, "155": 336, "016": 337, "TIC": 338, "PDI": 339, "014": 340, "1502": 341, "156": 342, "1555": 343, "1522": 344, "1506": 345, "1567": 346, "CI": 347, "HCI": 348, "UA": 349, "HA": 350, "020": 351, "027": 352, "AIC": 353, "FAL": 354, "FALL": 355, "037": 356, "043": 357, "BAL": 358, "34": 359, "1734": 360, "042": 361, "1520": 362, "69": 363, "1769": 364, "009": 365, "1507": 366, "1001": 367, "41": 368, "017": 369, "171": 370, "FD": 371, "022": 372, "012": 373, "1710": 374, "1518": 375, "036": 376, "CE": 377, "MS": 378, "SO": 379, "CEMS": 380, "1509": 381, "1501": 382, "416": 383, "1504": 384, "414": 385, "035": 386, "1523": 387, "2021": 388, "HH": 389, "1701": 390, "AM": 391, "CC": 392, "MCC": 393, "PS": 394, "PMCC": 395, "AMPS": 396, "154": 397, "1521": 398, "1551": 399, "041": 400, "006": 401, "172": 402, "TX": 403, "159": 404, "TXI": 405, "FFIC": 406, "TAH": 407, "1005": 408, "1515": 409, "BA": 410, "FF": 411, "FFX": 412, "1723": 413, "024": 414, "418": 415, "1519": 416, "1513": 417, "1594": 418, "1533": 419, "1543": 420, "204": 421, "1510": 422, "2040": 423, "12": 424, "1529": 425, "202": 426, "2025": 427, "1511": 428, "1003": 429, "AAH": 430, "1540": 431, "170": 432, "AMPSA": 433, "AMPSC": 434, "1705": 435, "AMPSB": 436, "1516": 437, "1563": 438, "16": 439, "31": 440, "1631": 441, "1711": 442, "1534": 443, "018": 444, "046": 445, "1514": 446, "1592": 447, "1760": 448, "1508": 449, "1724": 450, "1525": 451, "038": 452, "08": 453, "808": 454, "606": 455, "618": 456, "1527": 457 }, "merges": [ "Ġ 4", "Ġ4 7", "0 0", "A N", "AN N", "A L", "00 3", "T I", "Ġ4 8", "P AL", "00 7", "0 3", "L I", "F LI", "03 2", "A B", "T AL", "00 2", "F I", "FI C", "03 4", "1 5", "A H", "AH H", "0 2", "2 003", "0 1", "L AHH", "S I", "9 00", "8 02", "U SI", "7 0", "8 70", "15 1", "P I", "151 2", "4 7", "A I", "03 9", "00 1", "15 03", "5 0", "8 50", "X L", "LI C", "00 8", "PAL L", "01 0", "6 0", "01 1", "P D", "1 8", "AB C", "X A", "SI O", "60 5", "PD AH", "Ġ SIO", "00 4", "A AHH", "02 1", "15 0", "C D", "00 5", "4 8", "15 2", "1 7", "01 9", "F S", "FS L", "PI C", "0 4", "15 3", "60 8", "153 5", "0 15", "01 3", "150 5", "15 5", "01 6", "TI C", "PD I", "01 4", "15 02", "15 6", "155 5", "152 2", "150 6", "156 7", "C I", "H CI", "U A", "H A", "02 0", "02 7", "AI C", "F AL", "FAL L", "03 7", "04 3", "B AL", "3 4", "17 34", "04 2", "152 0", "6 9", "17 69", "00 9", "150 7", "1 001", "4 1", "01 7", "17 1", "F D", "02 2", "01 2", "171 0", "151 8", "03 6", "C E", "M S", "S O", "CE MS", "150 9", "15 01", "41 6", "150 4", "41 4", "03 5", "152 3", "2 021", "H H", "17 01", "A M", "C C", "M CC", "P S", "P MCC", "AM PS", "15 4", "152 1", "155 1", "04 1", "00 6", "17 2", "T X", "15 9", "TX I", "F FIC", "T AH", "1 005", "15 15", "B A", "F F", "FF X", "172 3", "02 4", "4 18", "151 9", "151 3", "159 4", "153 3", "154 3", "2 04", "151 0", "204 0", "1 2", "152 9", "2 02", "202 5", "151 1", "1 003", "A AH", "154 0", "1 70", "AMPS A", "AMPS C", "170 5", "AMPS B", "151 6", "156 3", "1 6", "3 1", "16 31", "171 1", "153 4", "01 8", "04 6", "151 4", "159 2", "17 60", "150 8", "172 4", "152 5", "03 8", "0 8", "8 08", "60 6", "6 18", "152 7" ] } }