4xF-47m-init / added_tokens.json
crumb's picture
Upload tokenizer
ab7d2cc
{
"<EXTRA_ID_0>": 32000,
"<EXTRA_ID_10>": 32010,
"<EXTRA_ID_11>": 32011,
"<EXTRA_ID_12>": 32012,
"<EXTRA_ID_13>": 32013,
"<EXTRA_ID_14>": 32014,
"<EXTRA_ID_15>": 32015,
"<EXTRA_ID_16>": 32016,
"<EXTRA_ID_17>": 32017,
"<EXTRA_ID_18>": 32018,
"<EXTRA_ID_19>": 32019,
"<EXTRA_ID_1>": 32001,
"<EXTRA_ID_20>": 32020,
"<EXTRA_ID_21>": 32021,
"<EXTRA_ID_22>": 32022,
"<EXTRA_ID_23>": 32023,
"<EXTRA_ID_24>": 32024,
"<EXTRA_ID_25>": 32025,
"<EXTRA_ID_26>": 32026,
"<EXTRA_ID_27>": 32027,
"<EXTRA_ID_28>": 32028,
"<EXTRA_ID_29>": 32029,
"<EXTRA_ID_2>": 32002,
"<EXTRA_ID_30>": 32030,
"<EXTRA_ID_31>": 32031,
"<EXTRA_ID_32>": 32032,
"<EXTRA_ID_33>": 32033,
"<EXTRA_ID_34>": 32034,
"<EXTRA_ID_35>": 32035,
"<EXTRA_ID_36>": 32036,
"<EXTRA_ID_37>": 32037,
"<EXTRA_ID_38>": 32038,
"<EXTRA_ID_39>": 32039,
"<EXTRA_ID_3>": 32003,
"<EXTRA_ID_40>": 32040,
"<EXTRA_ID_41>": 32041,
"<EXTRA_ID_42>": 32042,
"<EXTRA_ID_43>": 32043,
"<EXTRA_ID_44>": 32044,
"<EXTRA_ID_45>": 32045,
"<EXTRA_ID_46>": 32046,
"<EXTRA_ID_47>": 32047,
"<EXTRA_ID_48>": 32048,
"<EXTRA_ID_49>": 32049,
"<EXTRA_ID_4>": 32004,
"<EXTRA_ID_50>": 32050,
"<EXTRA_ID_51>": 32051,
"<EXTRA_ID_52>": 32052,
"<EXTRA_ID_53>": 32053,
"<EXTRA_ID_54>": 32054,
"<EXTRA_ID_55>": 32055,
"<EXTRA_ID_56>": 32056,
"<EXTRA_ID_57>": 32057,
"<EXTRA_ID_58>": 32058,
"<EXTRA_ID_59>": 32059,
"<EXTRA_ID_5>": 32005,
"<EXTRA_ID_60>": 32060,
"<EXTRA_ID_61>": 32061,
"<EXTRA_ID_62>": 32062,
"<EXTRA_ID_63>": 32063,
"<EXTRA_ID_6>": 32006,
"<EXTRA_ID_7>": 32007,
"<EXTRA_ID_8>": 32008,
"<EXTRA_ID_9>": 32009
}