|
{ |
|
"add_prefix_space": false, |
|
"added_tokens_decoder": { |
|
"151643": { |
|
"content": "<|endoftext|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151644": { |
|
"content": "<|im_start|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151645": { |
|
"content": "<|im_end|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151646": { |
|
"content": "<|object_ref_start|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151647": { |
|
"content": "<|object_ref_end|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151648": { |
|
"content": "<|box_start|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151649": { |
|
"content": "<|box_end|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151650": { |
|
"content": "<|quad_start|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151651": { |
|
"content": "<|quad_end|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151652": { |
|
"content": "<|vision_start|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151653": { |
|
"content": "<|vision_end|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151654": { |
|
"content": "<|vision_pad|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151655": { |
|
"content": "<|image_pad|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151656": { |
|
"content": "<|video_pad|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"151657": { |
|
"content": "<tool_call>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"151658": { |
|
"content": "</tool_call>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"151659": { |
|
"content": "<|fim_prefix|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"151660": { |
|
"content": "<|fim_middle|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"151661": { |
|
"content": "<|fim_suffix|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"151662": { |
|
"content": "<|fim_pad|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"151663": { |
|
"content": "<|repo_name|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
}, |
|
"151664": { |
|
"content": "<|file_sep|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": false |
|
} |
|
}, |
|
"additional_special_tokens": [ |
|
"<|im_start|>", |
|
"<|im_end|>" |
|
], |
|
"bos_token": null, |
|
"chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\n\n你是一个经过良好训练的AI助手,你的名字是Marco-o1.由阿里国际数字商业集团的AI Business创造.\n \n## 重要!!!!!\n当你回答问题时,你的思考应该在<Thought>内完成,<Output>内输出你的结果。\n<Thought>应该尽可能是英文,但是有2个特例,一个是对原文中的引用,另一个是是数学应该使用markdown格式,<Output>内的输出需要遵循用户输入的语言。\n <|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}", |
|
"clean_up_tokenization_spaces": false, |
|
"eos_token": "<|im_end|>", |
|
"errors": "replace", |
|
"max_length": 4096, |
|
"model_max_length": 131072, |
|
"pad_token": "<|endoftext|>", |
|
"split_special_tokens": false, |
|
"stride": 0, |
|
"tokenizer_class": "Qwen2Tokenizer", |
|
"truncation_side": "right", |
|
"truncation_strategy": "longest_first", |
|
"unk_token": null |
|
} |
|
|