msharma95 Raj-Sanjay-Shah commited on
Commit
8e3929c
·
1 Parent(s): cb76d2a

Update tokenizer_config.json (#3)

Browse files

- Update tokenizer_config.json (aaa29ea5f424350f4aa297d9c5dcb04ea4693cef)


Co-authored-by: Raj Sanjay Shah <Raj-Sanjay-Shah@users.noreply.huggingface.co>

Files changed (1) hide show
  1. tokenizer_config.json +2 -1
tokenizer_config.json CHANGED
@@ -1,4 +1,5 @@
1
  {
 
2
  "add_prefix_space": true,
3
  "added_tokens_decoder": {
4
  "0": {
@@ -61,4 +62,4 @@
61
  "truncation_side": "right",
62
  "truncation_strategy": "longest_first",
63
  "unk_token": "<unk>"
64
- }
 
1
  {
2
+ "errors": "replace",
3
  "add_prefix_space": true,
4
  "added_tokens_decoder": {
5
  "0": {
 
62
  "truncation_side": "right",
63
  "truncation_strategy": "longest_first",
64
  "unk_token": "<unk>"
65
+ }