Update preprocessor/tokenizer_config.yaml
Browse files
preprocessor/tokenizer_config.yaml
CHANGED
|
@@ -6,6 +6,7 @@ padding_strategy: longest
|
|
| 6 |
padding_direction: right
|
| 7 |
pad_token_id: 0
|
| 8 |
pad_token: <pad>
|
|
|
|
| 9 |
continuing_subword_prefix: ''
|
| 10 |
end_of_word_suffix: ''
|
| 11 |
fuse_unk: false
|
|
|
|
| 6 |
padding_direction: right
|
| 7 |
pad_token_id: 0
|
| 8 |
pad_token: <pad>
|
| 9 |
+
pad_token_type_id: 0
|
| 10 |
continuing_subword_prefix: ''
|
| 11 |
end_of_word_suffix: ''
|
| 12 |
fuse_unk: false
|