namisha commited on
Commit
606483b
1 Parent(s): 50c90fd

Upload processor

Browse files
Files changed (3) hide show
  1. added_tokens.json +2 -0
  2. special_tokens_map.json +2 -0
  3. tokenizer.json +18 -0
added_tokens.json CHANGED
@@ -1,5 +1,7 @@
1
  {
 
2
  "<s_iitcdip>": 57523,
 
3
  "<s_synthdog>": 57524,
4
  "<sep/>": 57522
5
  }
 
1
  {
2
+ "</s_label>": 57526,
3
  "<s_iitcdip>": 57523,
4
+ "<s_label>": 57525,
5
  "<s_synthdog>": 57524,
6
  "<sep/>": 57522
7
  }
special_tokens_map.json CHANGED
@@ -1,5 +1,7 @@
1
  {
2
  "additional_special_tokens": [
 
 
3
  "<s>",
4
  "</s>"
5
  ],
 
1
  {
2
  "additional_special_tokens": [
3
+ "<s_label>",
4
+ "</s_label>",
5
  "<s>",
6
  "</s>"
7
  ],
tokenizer.json CHANGED
@@ -88,6 +88,24 @@
88
  "rstrip": false,
89
  "normalized": false,
90
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
91
  }
92
  ],
93
  "normalizer": {
 
88
  "rstrip": false,
89
  "normalized": false,
90
  "special": true
91
+ },
92
+ {
93
+ "id": 57525,
94
+ "content": "<s_label>",
95
+ "single_word": false,
96
+ "lstrip": false,
97
+ "rstrip": false,
98
+ "normalized": false,
99
+ "special": true
100
+ },
101
+ {
102
+ "id": 57526,
103
+ "content": "</s_label>",
104
+ "single_word": false,
105
+ "lstrip": false,
106
+ "rstrip": false,
107
+ "normalized": false,
108
+ "special": true
109
  }
110
  ],
111
  "normalizer": {