{ "add_bos_token": true, "add_prefix_space": true, "added_tokens_decoder": { "15": { "content": "0", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "16": { "content": "1", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "17": { "content": "2", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "18": { "content": "3", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "19": { "content": "4", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "20": { "content": "5", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "21": { "content": "6", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "22": { "content": "7", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "23": { "content": "8", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "24": { "content": "9", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50256": { "content": "<|endoftext|>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": true }, "50257": { "content": "<|bos|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50258": { "content": "<|pad|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50259": { "content": "<|unk|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50260": { "content": "<|start_header_id|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50261": { "content": "<|end_header_id|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50262": { "content": "<|eot_id|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50263": { "content": "<|unused_1|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50264": { "content": "<|unused_2|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50265": { "content": "<|unused_3|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50266": { "content": "<|unused_4|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50267": { "content": "<|unused_5|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50268": { "content": "<|unused_6|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50269": { "content": "<|unused_7|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50270": { "content": "<|unused_8|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50271": { "content": "<|unused_9|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50272": { "content": "<|unused_10|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50273": { "content": "<|unused_11|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50274": { "content": "<|unused_12|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50275": { "content": "<|unused_13|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50276": { "content": "<|unused_14|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50277": { "content": "<|unused_15|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50278": { "content": "<|unused_16|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50279": { "content": "<|unused_17|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50280": { "content": "<|unused_18|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50281": { "content": "<|unused_19|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50282": { "content": "<|unused_20|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50283": { "content": "<|unused_21|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50284": { "content": "<|unused_22|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50285": { "content": "<|unused_23|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50286": { "content": "<|unused_24|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50287": { "content": "<|unused_25|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50288": { "content": "<|unused_26|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50289": { "content": "<|unused_27|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50290": { "content": "<|unused_28|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50291": { "content": "<|unused_29|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50292": { "content": "<|unused_30|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50293": { "content": "<|unused_31|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50294": { "content": "<|unused_32|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50295": { "content": "<|unused_33|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50296": { "content": "<|unused_34|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50297": { "content": "<|unused_35|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50298": { "content": "<|unused_36|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50299": { "content": "<|unused_37|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50300": { "content": "<|unused_38|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50301": { "content": "<|unused_39|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50302": { "content": "<|unused_40|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "50303": { "content": "<|unused_41|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true } }, "additional_special_tokens": [ "<|bos|>", "<|pad|>", "<|unk|>", "<|start_header_id|>", "<|end_header_id|>", "<|eot_id|>", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "<|unused_1|>", "<|unused_2|>", "<|unused_3|>", "<|unused_4|>", "<|unused_5|>", "<|unused_6|>", "<|unused_7|>", "<|unused_8|>", "<|unused_9|>", "<|unused_10|>", "<|unused_11|>", "<|unused_12|>", "<|unused_13|>", "<|unused_14|>", "<|unused_15|>", "<|unused_16|>", "<|unused_17|>", "<|unused_18|>", "<|unused_19|>", "<|unused_20|>", "<|unused_21|>", "<|unused_22|>", "<|unused_23|>", "<|unused_24|>", "<|unused_25|>", "<|unused_26|>", "<|unused_27|>", "<|unused_28|>", "<|unused_29|>", "<|unused_30|>", "<|unused_31|>", "<|unused_32|>", "<|unused_33|>", "<|unused_34|>", "<|unused_35|>", "<|unused_36|>", "<|unused_37|>", "<|unused_38|>", "<|unused_39|>", "<|unused_40|>", "<|unused_41|>" ], "bos_token": "<|bos|>", "clean_up_tokenization_spaces": true, "eos_token": "<|endoftext|>", "errors": "replace", "model_max_length": 4096, "pad_token": "<|pad|>", "padding_side": "left", "tokenizer_class": "GPT2Tokenizer", "unk_token": "<|unk|>" }