andrejonasson
commited on
Commit
•
3709246
1
Parent(s):
aa2e23e
Add bos and eos token to tokenizer config.
Browse fileshttps://huggingface.co/meta-llama/Llama-3.2-1B/blob/main/tokenizer_config.json
Update to be similar to llama tokenizer config.
- tokenizer_config.json +2 -0
tokenizer_config.json
CHANGED
@@ -33,7 +33,9 @@
|
|
33 |
"special": true
|
34 |
}
|
35 |
},
|
|
|
36 |
"clean_up_tokenization_spaces": true,
|
|
|
37 |
"model_max_length": 1000000000000000019884624838656,
|
38 |
"tokenizer_class": "PreTrainedTokenizerFast"
|
39 |
}
|
|
|
33 |
"special": true
|
34 |
}
|
35 |
},
|
36 |
+
"bos_token": "<|begin_of_text|>",
|
37 |
"clean_up_tokenization_spaces": true,
|
38 |
+
"eos_token": "<|end_of_text|>",
|
39 |
"model_max_length": 1000000000000000019884624838656,
|
40 |
"tokenizer_class": "PreTrainedTokenizerFast"
|
41 |
}
|