Upload tokenizer
Browse files- added_tokens.json +2 -2
- vocab.json +3 -8
added_tokens.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
{
|
| 2 |
-
"</s>":
|
| 3 |
-
"<s>":
|
| 4 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"</s>": 36,
|
| 3 |
+
"<s>": 35
|
| 4 |
}
|
vocab.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
-
"[PAD]":
|
| 3 |
-
"[UNK]":
|
| 4 |
"|": 0,
|
| 5 |
"ุก": 1,
|
| 6 |
"ุข": 2,
|
|
@@ -33,10 +33,5 @@
|
|
| 33 |
"ู": 29,
|
| 34 |
"ู": 30,
|
| 35 |
"ู": 31,
|
| 36 |
-
"ู": 32
|
| 37 |
-
"ู": 33,
|
| 38 |
-
"ู": 34,
|
| 39 |
-
"ู": 35,
|
| 40 |
-
"ู": 36,
|
| 41 |
-
"ู": 37
|
| 42 |
}
|
|
|
|
| 1 |
{
|
| 2 |
+
"[PAD]": 34,
|
| 3 |
+
"[UNK]": 33,
|
| 4 |
"|": 0,
|
| 5 |
"ุก": 1,
|
| 6 |
"ุข": 2,
|
|
|
|
| 33 |
"ู": 29,
|
| 34 |
"ู": 30,
|
| 35 |
"ู": 31,
|
| 36 |
+
"ู": 32
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
}
|