nielsr HF staff commited on
Commit
3add0d9
1 Parent(s): 444ed7f

Update fast tokenizer files

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -2
tokenizer_config.json CHANGED
@@ -1207,7 +1207,7 @@
1207
  "extra_ids": 100,
1208
  "from_slow": true,
1209
  "loc_extra_ids": 501,
1210
- "model_max_length": 512,
1211
  "only_label_first_subword": true,
1212
  "other_extra_ids": 200,
1213
  "pad_token": "<pad>",
@@ -1218,7 +1218,6 @@
1218
  0
1219
  ],
1220
  "pad_token_label": -100,
1221
- "processor_class": "UdopProcessor",
1222
  "sep_token": "</s>",
1223
  "sep_token_box": [
1224
  1000,
 
1207
  "extra_ids": 100,
1208
  "from_slow": true,
1209
  "loc_extra_ids": 501,
1210
+ "model_max_length": 1000000000000000019884624838656,
1211
  "only_label_first_subword": true,
1212
  "other_extra_ids": 200,
1213
  "pad_token": "<pad>",
 
1218
  0
1219
  ],
1220
  "pad_token_label": -100,
 
1221
  "sep_token": "</s>",
1222
  "sep_token_box": [
1223
  1000,