Commit
•
3caa69f
1
Parent(s):
8ba0b1b
Update README.md
Browse files
README.md
CHANGED
|
@@ -3,7 +3,6 @@ from transformers import LlamaTokenizer
|
|
| 3 |
|
| 4 |
tokenizer = LlamaTokenizer.from_pretrained(
|
| 5 |
'ocisd4/openllama_tokenizer_ext_zh',
|
| 6 |
-
pad_token="<|pad|>",
|
| 7 |
add_bos_token=True,
|
| 8 |
add_eos_token=False,
|
| 9 |
use_auth_token='True',
|
|
@@ -33,4 +32,4 @@ print(tokenizer.decode(tokenizer.encode(text)))
|
|
| 33 |
|
| 34 |
### updated
|
| 35 |
#### 2023-06-02
|
| 36 |
-
- add special tokens: <|pad
|
|
|
|
| 3 |
|
| 4 |
tokenizer = LlamaTokenizer.from_pretrained(
|
| 5 |
'ocisd4/openllama_tokenizer_ext_zh',
|
|
|
|
| 6 |
add_bos_token=True,
|
| 7 |
add_eos_token=False,
|
| 8 |
use_auth_token='True',
|
|
|
|
| 32 |
|
| 33 |
### updated
|
| 34 |
#### 2023-06-02
|
| 35 |
+
- add special tokens: <|pad|>, <|output|>, <|input|>, <|sep|>, <|emb|>, <|rwd|>, <|ctx|>
|