{ "add_prefix_space": false, "added_tokens_decoder": { "100256": { "content": "<|dummy_0|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100257": { "content": "<|endoftext|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100258": { "content": "<|fim_prefix|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100259": { "content": "<|fim_middle|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100260": { "content": "<|fim_suffix|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100261": { "content": "<|dummy_1|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100262": { "content": "<|dummy_2|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100263": { "content": "<|dummy_3|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100264": { "content": "<|im_start|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100265": { "content": "<|im_end|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100266": { "content": "<|im_sep|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100267": { "content": "<|dummy_4|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100268": { "content": "<|dummy_5|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100269": { "content": "<|dummy_6|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100270": { "content": "<|dummy_7|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100271": { "content": "<|dummy_8|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100272": { "content": "<|dummy_9|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100273": { "content": "<|dummy_10|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100274": { "content": "<|dummy_11|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100275": { "content": "<|dummy_12|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100276": { "content": "<|endofprompt|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100277": { "content": "<|dummy_13|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100278": { "content": "<|dummy_14|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100279": { "content": "<|dummy_15|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100280": { "content": "<|dummy_16|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100281": { "content": "<|dummy_17|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100282": { "content": "<|dummy_18|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100283": { "content": "<|dummy_19|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100284": { "content": "<|dummy_20|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100285": { "content": "<|dummy_21|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100286": { "content": "<|dummy_22|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100287": { "content": "<|dummy_23|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100288": { "content": "<|dummy_24|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100289": { "content": "<|dummy_25|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100290": { "content": "<|dummy_26|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100291": { "content": "<|dummy_27|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100292": { "content": "<|dummy_28|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100293": { "content": "<|dummy_29|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100294": { "content": "<|dummy_30|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100295": { "content": "<|dummy_31|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100296": { "content": "<|dummy_32|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100297": { "content": "<|dummy_33|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100298": { "content": "<|dummy_34|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100299": { "content": "<|dummy_35|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100300": { "content": "<|dummy_36|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100301": { "content": "<|dummy_37|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100302": { "content": "<|dummy_38|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100303": { "content": "<|dummy_39|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100304": { "content": "<|dummy_40|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100305": { "content": "<|dummy_41|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100306": { "content": "<|dummy_42|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100307": { "content": "<|dummy_43|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100308": { "content": "<|dummy_44|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100309": { "content": "<|dummy_45|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100310": { "content": "<|dummy_46|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100311": { "content": "<|dummy_47|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100312": { "content": "<|dummy_48|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100313": { "content": "<|dummy_49|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100314": { "content": "<|dummy_50|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100315": { "content": "<|dummy_51|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100316": { "content": "<|dummy_52|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100317": { "content": "<|dummy_53|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100318": { "content": "<|dummy_54|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100319": { "content": "<|dummy_55|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100320": { "content": "<|dummy_56|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100321": { "content": "<|dummy_57|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100322": { "content": "<|dummy_58|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100323": { "content": "<|dummy_59|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100324": { "content": "<|dummy_60|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100325": { "content": "<|dummy_61|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100326": { "content": "<|dummy_62|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100327": { "content": "<|dummy_63|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100328": { "content": "<|dummy_64|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100329": { "content": "<|dummy_65|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100330": { "content": "<|dummy_66|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100331": { "content": "<|dummy_67|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100332": { "content": "<|dummy_68|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100333": { "content": "<|dummy_69|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100334": { "content": "<|dummy_70|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100335": { "content": "<|dummy_71|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100336": { "content": "<|dummy_72|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100337": { "content": "<|dummy_73|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100338": { "content": "<|dummy_74|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100339": { "content": "<|dummy_75|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100340": { "content": "<|dummy_76|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100341": { "content": "<|dummy_77|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100342": { "content": "<|dummy_78|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100343": { "content": "<|dummy_79|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100344": { "content": "<|dummy_80|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100345": { "content": "<|dummy_81|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100346": { "content": "<|dummy_82|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100347": { "content": "<|dummy_83|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100348": { "content": "<|dummy_84|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100349": { "content": "<|dummy_85|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100350": { "content": "<|dummy_86|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true }, "100351": { "content": "<|dummy_87|>", "lstrip": true, "normalized": false, "rstrip": true, "single_word": false, "special": true } }, "bos_token": "<|endoftext|>", "chat_template": "{% for message in messages %}{% if (message['role'] == 'system') %}{{'<|im_start|>system<|im_sep|>' + message['content'] + '<|im_end|>'}}{% elif (message['role'] == 'user') %}{{'<|im_start|>user<|im_sep|>' + message['content'] + '<|im_end|><|im_start|>assistant<|im_sep|>'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|im_end|>'}}{% endif %}{% endfor %}", "clean_up_tokenization_spaces": false, "eos_token": "<|endoftext|>", "model_max_length": 16384, "pad_token": "<|endoftext|>", "tokenizer_class": "GPT2Tokenizer" }