new tokenizer
Browse files- special_tokens_map.json +3 -0
- spiece.model +3 -0
- tokenizer.json +3 -0
- tokenizer_config.json +3 -0
special_tokens_map.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:09059cedc26bc46bc09a52f05b92d4922e11917e87f3b92059bb1a63a59ab2c4
|
| 3 |
+
size 65
|
spiece.model
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ef78f86560d809067d12bac6c09f19a462cb3af3f54d2b8acbba26e1433125d6
|
| 3 |
+
size 4309802
|
tokenizer.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:da6bba5b70ae3accc6d2785bdacf64ab6f6fe9a8563079e6824ede7c7f2ab7a4
|
| 3 |
+
size 8326275
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ed4f9fa00c4248c8922aed5f185ad8c6a06e147807c44cf1998195f6dab6c2cb
|
| 3 |
+
size 412
|