ApostolosK commited on
Commit
5350d69
·
verified ·
1 Parent(s): a507774

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +9 -3
tokenizer_config.json CHANGED
@@ -1,5 +1,11 @@
1
  {
 
 
2
  "model_max_length": 512,
3
- "padding_side": "right",
4
- "truncation_side": "right"
5
- }
 
 
 
 
 
1
  {
2
+ "tokenizer_class": "DebertaV2Tokenizer",
3
+ "vocab_file": "sentencepiece.model",
4
  "model_max_length": 512,
5
+ "do_lower_case": false,
6
+ "special_tokens_map_file": null,
7
+ "unk_token": "<unk>",
8
+ "pad_token": "<pad>",
9
+ "bos_token": "<s>",
10
+ "eos_token": "</s>"
11
+ }