ykhwang commited on
Commit
d63893d
1 Parent(s): 53c8d7b

Fixed tokenizer_config.json

Browse files

Fixed tokenizer_config.json's inconsistent BOS, UNK token contents. Also added a `padding_side` configuration as same as SFT's tokenizer.

Files changed (1) hide show
  1. tokenizer_config.json +3 -2
tokenizer_config.json CHANGED
@@ -3,7 +3,7 @@
3
  "add_prefix_space": false,
4
  "bos_token": {
5
  "__type": "AddedToken",
6
- "content": "<|endoftext|>",
7
  "lstrip": false,
8
  "normalized": true,
9
  "rstrip": false,
@@ -21,10 +21,11 @@
21
  "errors": "replace",
22
  "model_max_length": 8192,
23
  "pad_token": null,
 
24
  "tokenizer_class": "GPT2Tokenizer",
25
  "unk_token": {
26
  "__type": "AddedToken",
27
- "content": "<|endoftext|>",
28
  "lstrip": false,
29
  "normalized": true,
30
  "rstrip": false,
 
3
  "add_prefix_space": false,
4
  "bos_token": {
5
  "__type": "AddedToken",
6
+ "content": "<||bos||>",
7
  "lstrip": false,
8
  "normalized": true,
9
  "rstrip": false,
 
21
  "errors": "replace",
22
  "model_max_length": 8192,
23
  "pad_token": null,
24
+ "padding_side": "right",
25
  "tokenizer_class": "GPT2Tokenizer",
26
  "unk_token": {
27
  "__type": "AddedToken",
28
+ "content": "<||unk||>",
29
  "lstrip": false,
30
  "normalized": true,
31
  "rstrip": false,