Hugging Face's logo

Meshrif
/
meshrif-model-sentimet-analysis

Text Classification
Transformers
TensorBoard
Safetensors
bert
Generated from Trainer
Inference Endpoints
Model card Files Metrics Community
meshrif-model-sentimet-analysis / tokenizer_config.json
Meshrif's picture
Meshrif
Training in progress, step 500
271a4b7 verified
raw
history blame
6.79 kB
{
"added_tokens_decoder": {
"0": {
"content": "+ุง",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"1": {
"content": "+ูƒ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"2": {
"content": "ุจ+",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"3": {
"content": "+ู‡ู…",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"4": {
"content": "+ุงุช",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"5": {
"content": "+ูŠ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"6": {
"content": "ู„+",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"7": {
"content": "+ู‡ู…ุง",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"8": {
"content": "+ู†ุง",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"9": {
"content": "+ู†",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"10": {
"content": "+ู‡ุง",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"11": {
"content": "+ูƒู…ุง",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"12": {
"content": "+ุฉ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"13": {
"content": "ู+",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"14": {
"content": "+ูƒู…",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"15": {
"content": "+ูƒู†",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"16": {
"content": "+ุช",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"17": {
"content": "[ุจุฑูŠุฏ]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"18": {
"content": "[ู…ุณุชุฎุฏู…]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"19": {
"content": "ู„ู„+",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"20": {
"content": "ุงู„+",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"21": {
"content": "[ุฑุงุจุท]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"22": {
"content": "ุณ+",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"23": {
"content": "+ุงู†",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"24": {
"content": "+ูˆุง",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"25": {
"content": "+ู‡",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"26": {
"content": "+ูˆู†",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"27": {
"content": "+ู‡ู†",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"28": {
"content": "+ูŠู†",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"29": {
"content": "ูˆ+",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"30": {
"content": "ูƒ+",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": true,
"special": true
},
"31": {
"content": "[PAD]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"32": {
"content": "[UNK]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"33": {
"content": "[CLS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"34": {
"content": "[SEP]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"35": {
"content": "[MASK]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
}
},
"clean_up_tokenization_spaces": true,
"cls_token": "[CLS]",
"do_basic_tokenize": true,
"do_lower_case": false,
"mask_token": "[MASK]",
"max_len": 512,
"model_max_length": 512,
"never_split": [
"+ูƒ",
"+ูƒู…ุง",
"ูƒ+",
"+ูˆุง",
"+ูŠู†",
"ูˆ+",
"+ูƒู†",
"+ุงู†",
"+ู‡ู…",
"+ุฉ",
"[ุจุฑูŠุฏ]",
"ู„ู„+",
"+ูŠ",
"+ุช",
"+ู†",
"ุณ+",
"ู„+",
"[ู…ุณุชุฎุฏู…]",
"+ูƒู…",
"+ุง",
"ุจ+",
"ู+",
"+ู†ุง",
"+ู‡ุง",
"+ูˆู†",
"+ู‡ู…ุง",
"ุงู„+",
"+ู‡",
"+ู‡ู†",
"+ุงุช",
"[ุฑุงุจุท]"
],
"pad_token": "[PAD]",
"sep_token": "[SEP]",
"strip_accents": null,
"tokenize_chinese_chars": true,
"tokenizer_class": "BertTokenizer",
"unk_token": "[UNK]"
}