Wimflorijn
commited on
Commit
•
b0ac0df
1
Parent(s):
dc33d6e
Upload tokenizer
Browse files- tokenizer.json +2 -4
tokenizer.json
CHANGED
@@ -1023,8 +1023,7 @@
|
|
1023 |
{
|
1024 |
"type": "Metaspace",
|
1025 |
"replacement": "▁",
|
1026 |
-
"
|
1027 |
-
"split": true
|
1028 |
},
|
1029 |
{
|
1030 |
"type": "Digits",
|
@@ -1081,8 +1080,7 @@
|
|
1081 |
"decoder": {
|
1082 |
"type": "Metaspace",
|
1083 |
"replacement": "▁",
|
1084 |
-
"
|
1085 |
-
"split": true
|
1086 |
},
|
1087 |
"model": {
|
1088 |
"type": "Unigram",
|
|
|
1023 |
{
|
1024 |
"type": "Metaspace",
|
1025 |
"replacement": "▁",
|
1026 |
+
"add_prefix_space": true
|
|
|
1027 |
},
|
1028 |
{
|
1029 |
"type": "Digits",
|
|
|
1080 |
"decoder": {
|
1081 |
"type": "Metaspace",
|
1082 |
"replacement": "▁",
|
1083 |
+
"add_prefix_space": true
|
|
|
1084 |
},
|
1085 |
"model": {
|
1086 |
"type": "Unigram",
|