Fix tokenizer.json
Browse files- tokenizer.json +1 -10
tokenizer.json
CHANGED
@@ -32,15 +32,6 @@
|
|
32 |
},
|
33 |
{
|
34 |
"id": 32000,
|
35 |
-
"content": "<pad>",
|
36 |
-
"single_word": false,
|
37 |
-
"lstrip": false,
|
38 |
-
"rstrip": false,
|
39 |
-
"normalized": true,
|
40 |
-
"special": false
|
41 |
-
},
|
42 |
-
{
|
43 |
-
"id": 32001,
|
44 |
"content": "[PAD]",
|
45 |
"single_word": false,
|
46 |
"lstrip": false,
|
@@ -93406,4 +93397,4 @@
|
|
93406 |
"β ββββββββββββββ"
|
93407 |
]
|
93408 |
}
|
93409 |
-
}
|
|
|
32 |
},
|
33 |
{
|
34 |
"id": 32000,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
35 |
"content": "[PAD]",
|
36 |
"single_word": false,
|
37 |
"lstrip": false,
|
|
|
93397 |
"β ββββββββββββββ"
|
93398 |
]
|
93399 |
}
|
93400 |
+
}
|