haijian06 bartowski commited on
Commit
cbd06ee
1 Parent(s): 9fbfd24

Add <|im_start|> as a special token to tokenizer_config.json (#4)

Browse files

- Add <|im_start|> as a special token to tokenizer_config.json (d20f49278c006877741954898e3e4e546ea2b5b8)


Co-authored-by: Bartowski <bartowski@users.noreply.huggingface.co>

Files changed (1) hide show
  1. tokenizer_config.json +8 -0
tokenizer_config.json CHANGED
@@ -27,6 +27,14 @@
27
  "single_word": false,
28
  "special": true
29
  },
 
 
 
 
 
 
 
 
30
  "7": {
31
  "content": "<|im_end|>",
32
  "lstrip": false,
 
27
  "single_word": false,
28
  "special": true
29
  },
30
+ "6": {
31
+ "content": "<|im_start|>",
32
+ "lstrip": false,
33
+ "normalized": false,
34
+ "rstrip": false,
35
+ "single_word": false,
36
+ "special": true
37
+ },
38
  "7": {
39
  "content": "<|im_end|>",
40
  "lstrip": false,