clap-asm / special_tokens_map.json
hustcw's picture
Upload tokenizer
610bd27 verified
{
"additional_special_tokens": [
"INSTR1",
"INSTR2",
"INSTR3",
"INSTR4",
"INSTR5",
"INSTR6",
"INSTR7",
"INSTR8",
"INSTR9",
"INSTR10",
"INSTR11",
"INSTR12",
"INSTR13",
"INSTR14",
"INSTR15",
"INSTR16",
"INSTR17",
"INSTR18",
"INSTR19",
"INSTR20",
"INSTR21",
"INSTR22",
"INSTR23",
"INSTR24",
"INSTR25",
"INSTR26",
"INSTR27",
"INSTR28",
"INSTR29",
"INSTR30",
"INSTR31",
"INSTR32",
"INSTR33",
"INSTR34",
"INSTR35",
"INSTR36",
"INSTR37",
"INSTR38",
"INSTR39",
"INSTR40",
"INSTR41",
"INSTR42",
"INSTR43",
"INSTR44",
"INSTR45",
"INSTR46",
"INSTR47",
"INSTR48",
"INSTR49",
"INSTR50",
"INSTR51",
"INSTR52",
"INSTR53",
"INSTR54",
"INSTR55",
"INSTR56",
"INSTR57",
"INSTR58",
"INSTR59",
"INSTR60",
"INSTR61",
"INSTR62",
"INSTR63",
"INSTR64",
"INSTR65",
"INSTR66",
"INSTR67",
"INSTR68",
"INSTR69",
"INSTR70",
"INSTR71",
"INSTR72",
"INSTR73",
"INSTR74",
"INSTR75",
"INSTR76",
"INSTR77",
"INSTR78",
"INSTR79",
"INSTR80",
"INSTR81",
"INSTR82",
"INSTR83",
"INSTR84",
"INSTR85",
"INSTR86",
"INSTR87",
"INSTR88",
"INSTR89",
"INSTR90",
"INSTR91",
"INSTR92",
"INSTR93",
"INSTR94",
"INSTR95",
"INSTR96",
"INSTR97",
"INSTR98",
"INSTR99",
"INSTR100",
"INSTR101",
"INSTR102",
"INSTR103",
"INSTR104",
"INSTR105",
"INSTR106",
"INSTR107",
"INSTR108",
"INSTR109",
"INSTR110",
"INSTR111",
"INSTR112",
"INSTR113",
"INSTR114",
"INSTR115",
"INSTR116",
"INSTR117",
"INSTR118",
"INSTR119",
"INSTR120",
"INSTR121",
"INSTR122",
"INSTR123",
"INSTR124",
"INSTR125",
"INSTR126",
"INSTR127",
"INSTR128",
"INSTR129",
"INSTR130",
"INSTR131",
"INSTR132",
"INSTR133",
"INSTR134",
"INSTR135",
"INSTR136",
"INSTR137",
"INSTR138",
"INSTR139",
"INSTR140",
"INSTR141",
"INSTR142",
"INSTR143",
"INSTR144",
"INSTR145",
"INSTR146",
"INSTR147",
"INSTR148",
"INSTR149",
"INSTR150",
"INSTR151",
"INSTR152",
"INSTR153",
"INSTR154",
"INSTR155",
"INSTR156",
"INSTR157",
"INSTR158",
"INSTR159",
"INSTR160",
"INSTR161",
"INSTR162",
"INSTR163",
"INSTR164",
"INSTR165",
"INSTR166",
"INSTR167",
"INSTR168",
"INSTR169",
"INSTR170",
"INSTR171",
"INSTR172",
"INSTR173",
"INSTR174",
"INSTR175",
"INSTR176",
"INSTR177",
"INSTR178",
"INSTR179",
"INSTR180",
"INSTR181",
"INSTR182",
"INSTR183",
"INSTR184",
"INSTR185",
"INSTR186",
"INSTR187",
"INSTR188",
"INSTR189",
"INSTR190",
"INSTR191",
"INSTR192",
"INSTR193",
"INSTR194",
"INSTR195",
"INSTR196",
"INSTR197",
"INSTR198",
"INSTR199",
"INSTR200",
"INSTR201",
"INSTR202",
"INSTR203",
"INSTR204",
"INSTR205",
"INSTR206",
"INSTR207",
"INSTR208",
"INSTR209",
"INSTR210",
"INSTR211",
"INSTR212",
"INSTR213",
"INSTR214",
"INSTR215",
"INSTR216",
"INSTR217",
"INSTR218",
"INSTR219",
"INSTR220",
"INSTR221",
"INSTR222",
"INSTR223",
"INSTR224",
"INSTR225",
"INSTR226",
"INSTR227",
"INSTR228",
"INSTR229",
"INSTR230",
"INSTR231",
"INSTR232",
"INSTR233",
"INSTR234",
"INSTR235",
"INSTR236",
"INSTR237",
"INSTR238",
"INSTR239",
"INSTR240",
"INSTR241",
"INSTR242",
"INSTR243",
"INSTR244",
"INSTR245",
"INSTR246",
"INSTR247",
"INSTR248",
"INSTR249",
"INSTR250",
"INSTR251",
"INSTR252",
"INSTR253",
"INSTR254",
"INSTR255",
"INSTR256",
"INSTR257",
"INSTR258",
"INSTR259",
"INSTR260",
"INSTR261",
"INSTR262",
"INSTR263",
"INSTR264",
"INSTR265",
"INSTR266",
"INSTR267",
"INSTR268",
"INSTR269",
"INSTR270",
"INSTR271",
"INSTR272",
"INSTR273",
"INSTR274",
"INSTR275",
"INSTR276",
"INSTR277",
"INSTR278",
"INSTR279",
"INSTR280",
"INSTR281",
"INSTR282",
"INSTR283",
"INSTR284",
"INSTR285",
"INSTR286",
"INSTR287",
"INSTR288",
"INSTR289",
"INSTR290",
"INSTR291",
"INSTR292",
"INSTR293",
"INSTR294",
"INSTR295",
"INSTR296",
"INSTR297",
"INSTR298",
"INSTR299",
"INSTR300",
"INSTR301",
"INSTR302",
"INSTR303",
"INSTR304",
"INSTR305",
"INSTR306",
"INSTR307",
"INSTR308",
"INSTR309",
"INSTR310",
"INSTR311",
"INSTR312",
"INSTR313",
"INSTR314",
"INSTR315",
"INSTR316",
"INSTR317",
"INSTR318",
"INSTR319",
"INSTR320",
"INSTR321",
"INSTR322",
"INSTR323",
"INSTR324",
"INSTR325",
"INSTR326",
"INSTR327",
"INSTR328",
"INSTR329",
"INSTR330",
"INSTR331",
"INSTR332",
"INSTR333",
"INSTR334",
"INSTR335",
"INSTR336",
"INSTR337",
"INSTR338",
"INSTR339",
"INSTR340",
"INSTR341",
"INSTR342",
"INSTR343",
"INSTR344",
"INSTR345",
"INSTR346",
"INSTR347",
"INSTR348",
"INSTR349",
"INSTR350",
"INSTR351",
"INSTR352",
"INSTR353",
"INSTR354",
"INSTR355",
"INSTR356",
"INSTR357",
"INSTR358",
"INSTR359",
"INSTR360",
"INSTR361",
"INSTR362",
"INSTR363",
"INSTR364",
"INSTR365",
"INSTR366",
"INSTR367",
"INSTR368",
"INSTR369",
"INSTR370",
"INSTR371",
"INSTR372",
"INSTR373",
"INSTR374",
"INSTR375",
"INSTR376",
"INSTR377",
"INSTR378",
"INSTR379",
"INSTR380",
"INSTR381",
"INSTR382",
"INSTR383",
"INSTR384",
"INSTR385",
"INSTR386",
"INSTR387",
"INSTR388",
"INSTR389",
"INSTR390",
"INSTR391",
"INSTR392",
"INSTR393",
"INSTR394",
"INSTR395",
"INSTR396",
"INSTR397",
"INSTR398",
"INSTR399",
"INSTR400",
"INSTR401",
"INSTR402",
"INSTR403",
"INSTR404",
"INSTR405",
"INSTR406",
"INSTR407",
"INSTR408",
"INSTR409",
"INSTR410",
"INSTR411",
"INSTR412",
"INSTR413",
"INSTR414",
"INSTR415",
"INSTR416",
"INSTR417",
"INSTR418",
"INSTR419",
"INSTR420",
"INSTR421",
"INSTR422",
"INSTR423",
"INSTR424",
"INSTR425",
"INSTR426",
"INSTR427",
"INSTR428",
"INSTR429",
"INSTR430",
"INSTR431",
"INSTR432",
"INSTR433",
"INSTR434",
"INSTR435",
"INSTR436",
"INSTR437",
"INSTR438",
"INSTR439",
"INSTR440",
"INSTR441",
"INSTR442",
"INSTR443",
"INSTR444",
"INSTR445",
"INSTR446",
"INSTR447",
"INSTR448",
"INSTR449",
"INSTR450",
"INSTR451",
"INSTR452",
"INSTR453",
"INSTR454",
"INSTR455",
"INSTR456",
"INSTR457",
"INSTR458",
"INSTR459",
"INSTR460",
"INSTR461",
"INSTR462",
"INSTR463",
"INSTR464",
"INSTR465",
"INSTR466",
"INSTR467",
"INSTR468",
"INSTR469",
"INSTR470",
"INSTR471",
"INSTR472",
"INSTR473",
"INSTR474",
"INSTR475",
"INSTR476",
"INSTR477",
"INSTR478",
"INSTR479",
"INSTR480",
"INSTR481",
"INSTR482",
"INSTR483",
"INSTR484",
"INSTR485",
"INSTR486",
"INSTR487",
"INSTR488",
"INSTR489",
"INSTR490",
"INSTR491",
"INSTR492",
"INSTR493",
"INSTR494",
"INSTR495",
"INSTR496",
"INSTR497",
"INSTR498",
"INSTR499",
"INSTR500",
"INSTR501",
"INSTR502",
"INSTR503",
"INSTR504",
"INSTR505",
"INSTR506",
"INSTR507",
"INSTR508",
"INSTR509",
"INSTR510",
"INSTR511",
"INSTR512",
"INSTR513",
"INSTR514",
"INSTR515",
"INSTR516",
"INSTR517",
"INSTR518",
"INSTR519",
"INSTR520",
"INSTR521",
"INSTR522",
"INSTR523",
"INSTR524",
"INSTR525",
"INSTR526",
"INSTR527",
"INSTR528",
"INSTR529",
"INSTR530",
"INSTR531",
"INSTR532",
"INSTR533",
"INSTR534",
"INSTR535",
"INSTR536",
"INSTR537",
"INSTR538",
"INSTR539",
"INSTR540",
"INSTR541",
"INSTR542",
"INSTR543",
"INSTR544",
"INSTR545",
"INSTR546",
"INSTR547",
"INSTR548",
"INSTR549",
"INSTR550",
"INSTR551",
"INSTR552",
"INSTR553",
"INSTR554",
"INSTR555",
"INSTR556",
"INSTR557",
"INSTR558",
"INSTR559",
"INSTR560",
"INSTR561",
"INSTR562",
"INSTR563",
"INSTR564",
"INSTR565",
"INSTR566",
"INSTR567",
"INSTR568",
"INSTR569",
"INSTR570",
"INSTR571",
"INSTR572",
"INSTR573",
"INSTR574",
"INSTR575",
"INSTR576",
"INSTR577",
"INSTR578",
"INSTR579",
"INSTR580",
"INSTR581",
"INSTR582",
"INSTR583",
"INSTR584",
"INSTR585",
"INSTR586",
"INSTR587",
"INSTR588",
"INSTR589",
"INSTR590",
"INSTR591",
"INSTR592",
"INSTR593",
"INSTR594",
"INSTR595",
"INSTR596",
"INSTR597",
"INSTR598",
"INSTR599",
"INSTR600",
"INSTR601",
"INSTR602",
"INSTR603",
"INSTR604",
"INSTR605",
"INSTR606",
"INSTR607",
"INSTR608",
"INSTR609",
"INSTR610",
"INSTR611",
"INSTR612",
"INSTR613",
"INSTR614",
"INSTR615",
"INSTR616",
"INSTR617",
"INSTR618",
"INSTR619",
"INSTR620",
"INSTR621",
"INSTR622",
"INSTR623",
"INSTR624",
"INSTR625",
"INSTR626",
"INSTR627",
"INSTR628",
"INSTR629",
"INSTR630",
"INSTR631",
"INSTR632",
"INSTR633",
"INSTR634",
"INSTR635",
"INSTR636",
"INSTR637",
"INSTR638",
"INSTR639",
"INSTR640",
"INSTR641",
"INSTR642",
"INSTR643",
"INSTR644",
"INSTR645",
"INSTR646",
"INSTR647",
"INSTR648",
"INSTR649",
"INSTR650",
"INSTR651",
"INSTR652",
"INSTR653",
"INSTR654",
"INSTR655",
"INSTR656",
"INSTR657",
"INSTR658",
"INSTR659",
"INSTR660",
"INSTR661",
"INSTR662",
"INSTR663",
"INSTR664",
"INSTR665",
"INSTR666",
"INSTR667",
"INSTR668",
"INSTR669",
"INSTR670",
"INSTR671",
"INSTR672",
"INSTR673",
"INSTR674",
"INSTR675",
"INSTR676",
"INSTR677",
"INSTR678",
"INSTR679",
"INSTR680",
"INSTR681",
"INSTR682",
"INSTR683",
"INSTR684",
"INSTR685",
"INSTR686",
"INSTR687",
"INSTR688",
"INSTR689",
"INSTR690",
"INSTR691",
"INSTR692",
"INSTR693",
"INSTR694",
"INSTR695",
"INSTR696",
"INSTR697",
"INSTR698",
"INSTR699",
"INSTR700",
"INSTR701",
"INSTR702",
"INSTR703",
"INSTR704",
"INSTR705",
"INSTR706",
"INSTR707",
"INSTR708",
"INSTR709",
"INSTR710",
"INSTR711",
"INSTR712",
"INSTR713",
"INSTR714",
"INSTR715",
"INSTR716",
"INSTR717",
"INSTR718",
"INSTR719",
"INSTR720",
"INSTR721",
"INSTR722",
"INSTR723",
"INSTR724",
"INSTR725",
"INSTR726",
"INSTR727",
"INSTR728",
"INSTR729",
"INSTR730",
"INSTR731",
"INSTR732",
"INSTR733",
"INSTR734",
"INSTR735",
"INSTR736",
"INSTR737",
"INSTR738",
"INSTR739",
"INSTR740",
"INSTR741",
"INSTR742",
"INSTR743",
"INSTR744",
"INSTR745",
"INSTR746",
"INSTR747",
"INSTR748",
"INSTR749",
"INSTR750",
"INSTR751",
"INSTR752",
"INSTR753",
"INSTR754",
"INSTR755",
"INSTR756",
"INSTR757",
"INSTR758",
"INSTR759",
"INSTR760",
"INSTR761",
"INSTR762",
"INSTR763",
"INSTR764",
"INSTR765",
"INSTR766",
"INSTR767",
"INSTR768",
"INSTR769",
"INSTR770",
"INSTR771",
"INSTR772",
"INSTR773",
"INSTR774",
"INSTR775",
"INSTR776",
"INSTR777",
"INSTR778",
"INSTR779",
"INSTR780",
"INSTR781",
"INSTR782",
"INSTR783",
"INSTR784",
"INSTR785",
"INSTR786",
"INSTR787",
"INSTR788",
"INSTR789",
"INSTR790",
"INSTR791",
"INSTR792",
"INSTR793",
"INSTR794",
"INSTR795",
"INSTR796",
"INSTR797",
"INSTR798",
"INSTR799",
"INSTR800",
"INSTR801",
"INSTR802",
"INSTR803",
"INSTR804",
"INSTR805",
"INSTR806",
"INSTR807",
"INSTR808",
"INSTR809",
"INSTR810",
"INSTR811",
"INSTR812",
"INSTR813",
"INSTR814",
"INSTR815",
"INSTR816",
"INSTR817",
"INSTR818",
"INSTR819",
"INSTR820",
"INSTR821",
"INSTR822",
"INSTR823",
"INSTR824",
"INSTR825",
"INSTR826",
"INSTR827",
"INSTR828",
"INSTR829",
"INSTR830",
"INSTR831",
"INSTR832",
"INSTR833",
"INSTR834",
"INSTR835",
"INSTR836",
"INSTR837",
"INSTR838",
"INSTR839",
"INSTR840",
"INSTR841",
"INSTR842",
"INSTR843",
"INSTR844",
"INSTR845",
"INSTR846",
"INSTR847",
"INSTR848",
"INSTR849",
"INSTR850",
"INSTR851",
"INSTR852",
"INSTR853",
"INSTR854",
"INSTR855",
"INSTR856",
"INSTR857",
"INSTR858",
"INSTR859",
"INSTR860",
"INSTR861",
"INSTR862",
"INSTR863",
"INSTR864",
"INSTR865",
"INSTR866",
"INSTR867",
"INSTR868",
"INSTR869",
"INSTR870",
"INSTR871",
"INSTR872",
"INSTR873",
"INSTR874",
"INSTR875",
"INSTR876",
"INSTR877",
"INSTR878",
"INSTR879",
"INSTR880",
"INSTR881",
"INSTR882",
"INSTR883",
"INSTR884",
"INSTR885",
"INSTR886",
"INSTR887",
"INSTR888",
"INSTR889",
"INSTR890",
"INSTR891",
"INSTR892",
"INSTR893",
"INSTR894",
"INSTR895",
"INSTR896",
"INSTR897",
"INSTR898",
"INSTR899",
"INSTR900",
"INSTR901",
"INSTR902",
"INSTR903",
"INSTR904",
"INSTR905",
"INSTR906",
"INSTR907",
"INSTR908",
"INSTR909",
"INSTR910",
"INSTR911",
"INSTR912",
"INSTR913",
"INSTR914",
"INSTR915",
"INSTR916",
"INSTR917",
"INSTR918",
"INSTR919",
"INSTR920",
"INSTR921",
"INSTR922",
"INSTR923",
"INSTR924",
"INSTR925",
"INSTR926",
"INSTR927",
"INSTR928",
"INSTR929",
"INSTR930",
"INSTR931",
"INSTR932",
"INSTR933",
"INSTR934",
"INSTR935",
"INSTR936",
"INSTR937",
"INSTR938",
"INSTR939",
"INSTR940",
"INSTR941",
"INSTR942",
"INSTR943",
"INSTR944",
"INSTR945",
"INSTR946",
"INSTR947",
"INSTR948",
"INSTR949",
"INSTR950",
"INSTR951",
"INSTR952",
"INSTR953",
"INSTR954",
"INSTR955",
"INSTR956",
"INSTR957",
"INSTR958",
"INSTR959",
"INSTR960",
"INSTR961",
"INSTR962",
"INSTR963",
"INSTR964",
"INSTR965",
"INSTR966",
"INSTR967",
"INSTR968",
"INSTR969",
"INSTR970",
"INSTR971",
"INSTR972",
"INSTR973",
"INSTR974",
"INSTR975",
"INSTR976",
"INSTR977",
"INSTR978",
"INSTR979",
"INSTR980",
"INSTR981",
"INSTR982",
"INSTR983",
"INSTR984",
"INSTR985",
"INSTR986",
"INSTR987",
"INSTR988",
"INSTR989",
"INSTR990",
"INSTR991",
"INSTR992",
"INSTR993",
"INSTR994",
"INSTR995",
"INSTR996",
"INSTR997",
"INSTR998",
"INSTR999",
"INSTR1000",
"INSTR1001",
"INSTR1002",
"INSTR1003",
"INSTR1004",
"INSTR1005",
"INSTR1006",
"INSTR1007",
"INSTR1008",
"INSTR1009",
"INSTR1010",
"INSTR1011",
"INSTR1012",
"INSTR1013",
"INSTR1014",
"INSTR1015",
"INSTR1016",
"INSTR1017",
"INSTR1018",
"INSTR1019",
"INSTR1020",
"INSTR1021",
"INSTR1022",
"INSTR1023",
"INSTR1024"
],
"bos_token": {
"content": "<s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"cls_token": {
"content": "<s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "</s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"mask_token": {
"content": "<mask>",
"lstrip": true,
"normalized": false,
"rstrip": false,
"single_word": false
},
"pad_token": {
"content": "<pad>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"sep_token": {
"content": "</s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"unk_token": {
"content": "[UNK]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
}
}