yentinglin minyichen commited on
Commit
f4b968f
1 Parent(s): 609a0eb

Update tokenizer_config.json (#4)

Browse files

- Upload tokenizer_config.json (263d7ce5e63f267f17db7a12c4b89f14c990b67f)
- Update tokenizer_config.json (72d02df0c343329511e796e7e67b5dbbf4a191fa)
- Update special_tokens_map.json (f7f8f66ce688863cfbe19514484a2f2b82080926)


Co-authored-by: minyi <minyichen@users.noreply.huggingface.co>

Files changed (2) hide show
  1. special_tokens_map.json +1 -1
  2. tokenizer_config.json +2 -2
special_tokens_map.json CHANGED
@@ -7,7 +7,7 @@
7
  "single_word": false
8
  },
9
  "eos_token": {
10
- "content": "<|im_end|>",
11
  "lstrip": false,
12
  "normalized": false,
13
  "rstrip": false,
 
7
  "single_word": false
8
  },
9
  "eos_token": {
10
+ "content": "<|eot_id|>",
11
  "lstrip": false,
12
  "normalized": false,
13
  "rstrip": false,
tokenizer_config.json CHANGED
@@ -2066,9 +2066,9 @@
2066
  }
2067
  },
2068
  "bos_token": "<|begin_of_text|>",
2069
- "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% else %}{{ eos_token }}{% endif %}",
2070
  "clean_up_tokenization_spaces": true,
2071
- "eos_token": "<|im_end|>",
2072
  "model_input_names": [
2073
  "input_ids",
2074
  "attention_mask"
 
2066
  }
2067
  },
2068
  "bos_token": "<|begin_of_text|>",
2069
+ "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}",
2070
  "clean_up_tokenization_spaces": true,
2071
+ "eos_token": "<|eot_id|>",
2072
  "model_input_names": [
2073
  "input_ids",
2074
  "attention_mask"