對tokenizer增加special tokens使其能被.apply_chat_template正確轉換

#34
Files changed (2) hide show
  1. special_tokens_map.json +1 -0
  2. tokenizer_config.json +47 -8
special_tokens_map.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {}
tokenizer_config.json CHANGED
@@ -1,13 +1,52 @@
1
  {
2
- "name_or_path": "THUDM/chatglm3-6b",
3
- "remove_space": false,
4
- "do_lower_case": false,
5
- "tokenizer_class": "ChatGLMTokenizer",
6
- "chat_template": "{% for message in messages %}{% if loop.first %}[gMASK]sop<|{{ message['role'] }}|> \n {{ message['content'] }}{% else %}<|{{ message['role'] }}|> \n {{ message['content'] }}{% endif %}{% endfor %}{% if add_generation_prompt %}<|assistant|>{% endif %}",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7
  "auto_map": {
8
  "AutoTokenizer": [
9
- "tokenization_chatglm.ChatGLMTokenizer",
10
  null
11
- ]
12
- }
 
 
 
 
 
 
 
 
 
 
13
  }
 
1
  {
2
+ "added_tokens_decoder": {
3
+ "64790": {
4
+ "content": "[gMASK]",
5
+ "lstrip": false,
6
+ "normalized": true,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": false
10
+ },
11
+ "64792": {
12
+ "content": "sop",
13
+ "lstrip": false,
14
+ "normalized": true,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": false
18
+ },
19
+ "64795": {
20
+ "content": "<|user|>",
21
+ "lstrip": false,
22
+ "normalized": true,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": false
26
+ },
27
+ "64796": {
28
+ "content": "<|assistant|>",
29
+ "lstrip": false,
30
+ "normalized": true,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": false
34
+ }
35
+ },
36
  "auto_map": {
37
  "AutoTokenizer": [
38
+ "THUDM/chatglm3-6b--tokenization_chatglm.ChatGLMTokenizer",
39
  null
40
+ ]
41
+ },
42
+ "chat_template": "{% for message in messages %}{% if loop.first %}[gMASK]sop<|{{ message['role'] }}|>\n {{ message['content'] }}{% else %}<|{{ message['role'] }}|> \n {{ message['content'] }}{% endif %}{% endfor %}{% if add_generation_prompt %}<|assistant|>{% endif %}",
43
+ "clean_up_tokenization_spaces": false,
44
+ "do_lower_case": false,
45
+ "eos_token": "</s>",
46
+ "model_max_length": 1000000000000000019884624838656,
47
+ "pad_token": "<unk>",
48
+ "padding_side": "left",
49
+ "remove_space": false,
50
+ "tokenizer_class": "ChatGLMTokenizer",
51
+ "unk_token": "<unk>"
52
  }