arxyzan commited on
Commit
7a1966d
1 Parent(s): b9289bf

Update preprocessor/tokenizer_config.yaml

Browse files
Files changed (1) hide show
  1. preprocessor/tokenizer_config.yaml +4 -4
preprocessor/tokenizer_config.yaml CHANGED
@@ -1,11 +1,11 @@
1
  name: bpe_tokenizer
2
  config_type: preprocessor
3
  max_length: 512
4
- truncation_strategy: longest_first
5
- truncation_direction: right
6
  stride: 0
7
- padding_strategy: longest
8
- padding_direction: right
9
  pad_to_multiple_of: 0
10
  pad_token_type_id: 0
11
  bos_token: <s>
 
1
  name: bpe_tokenizer
2
  config_type: preprocessor
3
  max_length: 512
4
+ truncation: longest_first
5
+ truncation_side: right
6
  stride: 0
7
+ padding: longest
8
+ padding_side: right
9
  pad_to_multiple_of: 0
10
  pad_token_type_id: 0
11
  bos_token: <s>