| { | |
| "add_prefix_space": false, | |
| "added_tokens_decoder": { | |
| "0": { | |
| "content": "<|endoftext|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "1": { | |
| "content": "<fim_prefix>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "2": { | |
| "content": "<fim_middle>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "3": { | |
| "content": "<fim_suffix>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "4": { | |
| "content": "<fim_pad>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "5": { | |
| "content": "<repo_name>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "6": { | |
| "content": "<file_sep>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "7": { | |
| "content": "<issue_start>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "8": { | |
| "content": "<issue_comment>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "9": { | |
| "content": "<issue_closed>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "10": { | |
| "content": "<jupyter_start>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "11": { | |
| "content": "<jupyter_text>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "12": { | |
| "content": "<jupyter_code>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "13": { | |
| "content": "<jupyter_output>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "14": { | |
| "content": "<jupyter_script>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "15": { | |
| "content": "<empty_output>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "16": { | |
| "content": "<code_to_intermediate>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "17": { | |
| "content": "<intermediate_to_code>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "18": { | |
| "content": "<pr>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "19": { | |
| "content": "<pr_status>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "20": { | |
| "content": "<pr_is_merged>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "21": { | |
| "content": "<pr_base>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "22": { | |
| "content": "<pr_file>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "23": { | |
| "content": "<pr_base_code>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "24": { | |
| "content": "<pr_diff>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "25": { | |
| "content": "<pr_diff_hunk>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "26": { | |
| "content": "<pr_comment>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "27": { | |
| "content": "<pr_event_id>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "28": { | |
| "content": "<pr_review>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "29": { | |
| "content": "<pr_review_state>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "30": { | |
| "content": "<pr_review_comment>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "31": { | |
| "content": "<pr_in_reply_to_review_id>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "32": { | |
| "content": "<pr_in_reply_to_comment_id>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "33": { | |
| "content": "<pr_diff_hunk_comment_line>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "34": { | |
| "content": "<NAME>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "35": { | |
| "content": "<EMAIL>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "36": { | |
| "content": "<KEY>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "37": { | |
| "content": "<PASSWORD>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| } | |
| }, | |
| "additional_special_tokens": [ | |
| "<|endoftext|>", | |
| "<fim_prefix>", | |
| "<fim_middle>", | |
| "<fim_suffix>", | |
| "<fim_pad>", | |
| "<repo_name>", | |
| "<file_sep>", | |
| "<issue_start>", | |
| "<issue_comment>", | |
| "<issue_closed>", | |
| "<jupyter_start>", | |
| "<jupyter_text>", | |
| "<jupyter_code>", | |
| "<jupyter_output>", | |
| "<jupyter_script>", | |
| "<empty_output>", | |
| "<code_to_intermediate>", | |
| "<intermediate_to_code>", | |
| "<pr>", | |
| "<pr_status>", | |
| "<pr_is_merged>", | |
| "<pr_base>", | |
| "<pr_file>", | |
| "<pr_base_code>", | |
| "<pr_diff>", | |
| "<pr_diff_hunk>", | |
| "<pr_comment>", | |
| "<pr_event_id>", | |
| "<pr_review>", | |
| "<pr_review_state>", | |
| "<pr_review_comment>", | |
| "<pr_in_reply_to_review_id>", | |
| "<pr_in_reply_to_comment_id>", | |
| "<pr_diff_hunk_comment_line>", | |
| "<NAME>", | |
| "<EMAIL>", | |
| "<KEY>", | |
| "<PASSWORD>" | |
| ], | |
| "bos_token": "<|endoftext|>", | |
| "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user' or message['role'] == 'human') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<s>system\n' + system_message + '\n' %}{% else %}{% set content = '' %}{% endif %}{% if message['role'] == 'user' or message['role'] == 'human' %}{{ content + '<s>user\n' + message['content'] + '\n' }}{% elif message['role'] == 'assistant' or message['role'] == 'bot' %}{{ '<s>assistant\n' + message['content'] + '\n' + eos_token + '\n'}}{% else %}{{ raise_exception('Only user/human and assistant/bot roles are supported!') }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<s>assistant\n' }}{% endif %}", | |
| "clean_up_tokenization_spaces": true, | |
| "eos_token": "<|endoftext|>", | |
| "model_max_length": 1000000000000000019884624838656, | |
| "tokenizer_class": "GPT2Tokenizer", | |
| "unk_token": "<|endoftext|>", | |
| "vocab_size": 49152 | |
| } | |