sanchit-gandhi HF staff commited on
Commit
b634ee6
1 Parent(s): 8a2a8a3

Upload processor

Browse files
Files changed (2) hide show
  1. special_tokens_map.json +1 -0
  2. tokenizer_config.json +2 -1
special_tokens_map.json CHANGED
@@ -122,6 +122,7 @@
122
  "rstrip": false,
123
  "single_word": false
124
  },
 
125
  "unk_token": {
126
  "content": "<|endoftext|>",
127
  "lstrip": false,
 
122
  "rstrip": false,
123
  "single_word": false
124
  },
125
+ "pad_token": "<|endoftext|>",
126
  "unk_token": {
127
  "content": "<|endoftext|>",
128
  "lstrip": false,
tokenizer_config.json CHANGED
@@ -12988,8 +12988,9 @@
12988
  "eos_token": "<|endoftext|>",
12989
  "errors": "replace",
12990
  "model_max_length": 1000000000000000019884624838656,
12991
- "pad_token": null,
12992
  "processor_class": "WhisperProcessor",
12993
  "tokenizer_class": "WhisperTokenizer",
 
12994
  "unk_token": "<|endoftext|>"
12995
  }
 
12988
  "eos_token": "<|endoftext|>",
12989
  "errors": "replace",
12990
  "model_max_length": 1000000000000000019884624838656,
12991
+ "pad_token": "<|endoftext|>",
12992
  "processor_class": "WhisperProcessor",
12993
  "tokenizer_class": "WhisperTokenizer",
12994
+ "trust_remote_code": false,
12995
  "unk_token": "<|endoftext|>"
12996
  }