office-dbModel-v0-3 / tokenizer /special_tokens_map.json
xarthurx's picture
8b7f85af4d840cf6d246f0d14be9b115e7ace775d68ff7a3a68afc8e9bb69e07
3c409df
raw
history blame
460 Bytes
{
"bos_token": {
"content": "<|startoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"pad_token": "!",
"unk_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
}