| { | |
| "added_tokens_decoder": { | |
| "0": { | |
| "content": "<s>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "1": { | |
| "content": "<pad>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "2": { | |
| "content": "</s>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "3": { | |
| "content": "<unk>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57521": { | |
| "content": "<mask>", | |
| "lstrip": true, | |
| "normalized": true, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57522": { | |
| "content": "<sep/>", | |
| "lstrip": false, | |
| "normalized": true, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": false | |
| }, | |
| "57523": { | |
| "content": "<s_iitcdip>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57524": { | |
| "content": "<s_synthdog>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57525": { | |
| "content": "<s_STATE_LIC_5>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57526": { | |
| "content": "</s_STATE_LIC_5>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57527": { | |
| "content": "<s_STATE_LIC_4>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57528": { | |
| "content": "</s_STATE_LIC_4>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57529": { | |
| "content": "<s_STATE_LIC_3>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57530": { | |
| "content": "</s_STATE_LIC_3>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57531": { | |
| "content": "<s_STATE_LIC_2>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57532": { | |
| "content": "</s_STATE_LIC_2>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57533": { | |
| "content": "<s_STATE_LIC_1>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57534": { | |
| "content": "</s_STATE_LIC_1>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57535": { | |
| "content": "<s_SEX_5>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57536": { | |
| "content": "</s_SEX_5>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57537": { | |
| "content": "<s_SEX_4>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57538": { | |
| "content": "</s_SEX_4>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57539": { | |
| "content": "<s_SEX_3>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57540": { | |
| "content": "</s_SEX_3>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57541": { | |
| "content": "<s_SEX_2>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57542": { | |
| "content": "</s_SEX_2>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57543": { | |
| "content": "<s_SEX_1>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57544": { | |
| "content": "</s_SEX_1>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57545": { | |
| "content": "<s_NAME_5>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57546": { | |
| "content": "</s_NAME_5>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57547": { | |
| "content": "<s_NAME_4>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57548": { | |
| "content": "</s_NAME_4>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57549": { | |
| "content": "<s_NAME_3>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57550": { | |
| "content": "</s_NAME_3>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57551": { | |
| "content": "<s_NAME_2>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57552": { | |
| "content": "</s_NAME_2>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57553": { | |
| "content": "<s_NAME_1>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57554": { | |
| "content": "</s_NAME_1>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57555": { | |
| "content": "<s_MAR_STAT_5>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57556": { | |
| "content": "</s_MAR_STAT_5>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57557": { | |
| "content": "<s_MAR_STAT_4>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57558": { | |
| "content": "</s_MAR_STAT_4>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57559": { | |
| "content": "<s_MAR_STAT_3>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57560": { | |
| "content": "</s_MAR_STAT_3>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57561": { | |
| "content": "<s_MAR_STAT_2>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57562": { | |
| "content": "</s_MAR_STAT_2>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57563": { | |
| "content": "<s_MAR_STAT_1>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57564": { | |
| "content": "</s_MAR_STAT_1>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57565": { | |
| "content": "<s_DRIVING_LICENCE_NUMBER_5>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57566": { | |
| "content": "</s_DRIVING_LICENCE_NUMBER_5>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57567": { | |
| "content": "<s_DRIVING_LICENCE_NUMBER_4>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57568": { | |
| "content": "</s_DRIVING_LICENCE_NUMBER_4>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57569": { | |
| "content": "<s_DRIVING_LICENCE_NUMBER_3>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57570": { | |
| "content": "</s_DRIVING_LICENCE_NUMBER_3>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57571": { | |
| "content": "<s_DRIVING_LICENCE_NUMBER_2>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57572": { | |
| "content": "</s_DRIVING_LICENCE_NUMBER_2>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57573": { | |
| "content": "<s_DRIVING_LICENCE_NUMBER_1>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57574": { | |
| "content": "</s_DRIVING_LICENCE_NUMBER_1>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57575": { | |
| "content": "<s_DRIVER_5>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57576": { | |
| "content": "</s_DRIVER_5>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57577": { | |
| "content": "<s_DRIVER_4>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57578": { | |
| "content": "</s_DRIVER_4>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57579": { | |
| "content": "<s_DRIVER_3>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57580": { | |
| "content": "</s_DRIVER_3>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57581": { | |
| "content": "<s_DRIVER_2>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57582": { | |
| "content": "</s_DRIVER_2>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57583": { | |
| "content": "<s_DRIVER_1>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57584": { | |
| "content": "</s_DRIVER_1>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57585": { | |
| "content": "<s_DATE_OF_BIRTH_formated_5>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57586": { | |
| "content": "</s_DATE_OF_BIRTH_formated_5>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57587": { | |
| "content": "<s_DATE_OF_BIRTH_formated_4>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57588": { | |
| "content": "</s_DATE_OF_BIRTH_formated_4>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57589": { | |
| "content": "<s_DATE_OF_BIRTH_formated_3>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57590": { | |
| "content": "</s_DATE_OF_BIRTH_formated_3>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57591": { | |
| "content": "<s_DATE_OF_BIRTH_formated_2>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57592": { | |
| "content": "</s_DATE_OF_BIRTH_formated_2>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57593": { | |
| "content": "<s_DATE_OF_BIRTH_formated_1>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "57594": { | |
| "content": "</s_DATE_OF_BIRTH_formated_1>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| } | |
| }, | |
| "additional_special_tokens": [ | |
| "<s_STATE_LIC_5>", | |
| "</s_STATE_LIC_5>", | |
| "<s_STATE_LIC_4>", | |
| "</s_STATE_LIC_4>", | |
| "<s_STATE_LIC_3>", | |
| "</s_STATE_LIC_3>", | |
| "<s_STATE_LIC_2>", | |
| "</s_STATE_LIC_2>", | |
| "<s_STATE_LIC_1>", | |
| "</s_STATE_LIC_1>", | |
| "<s_SEX_5>", | |
| "</s_SEX_5>", | |
| "<s_SEX_4>", | |
| "</s_SEX_4>", | |
| "<s_SEX_3>", | |
| "</s_SEX_3>", | |
| "<s_SEX_2>", | |
| "</s_SEX_2>", | |
| "<s_SEX_1>", | |
| "</s_SEX_1>", | |
| "<s_NAME_5>", | |
| "</s_NAME_5>", | |
| "<s_NAME_4>", | |
| "</s_NAME_4>", | |
| "<s_NAME_3>", | |
| "</s_NAME_3>", | |
| "<s_NAME_2>", | |
| "</s_NAME_2>", | |
| "<s_NAME_1>", | |
| "</s_NAME_1>", | |
| "<s_MAR_STAT_5>", | |
| "</s_MAR_STAT_5>", | |
| "<s_MAR_STAT_4>", | |
| "</s_MAR_STAT_4>", | |
| "<s_MAR_STAT_3>", | |
| "</s_MAR_STAT_3>", | |
| "<s_MAR_STAT_2>", | |
| "</s_MAR_STAT_2>", | |
| "<s_MAR_STAT_1>", | |
| "</s_MAR_STAT_1>", | |
| "<s_DRIVING_LICENCE_NUMBER_5>", | |
| "</s_DRIVING_LICENCE_NUMBER_5>", | |
| "<s_DRIVING_LICENCE_NUMBER_4>", | |
| "</s_DRIVING_LICENCE_NUMBER_4>", | |
| "<s_DRIVING_LICENCE_NUMBER_3>", | |
| "</s_DRIVING_LICENCE_NUMBER_3>", | |
| "<s_DRIVING_LICENCE_NUMBER_2>", | |
| "</s_DRIVING_LICENCE_NUMBER_2>", | |
| "<s_DRIVING_LICENCE_NUMBER_1>", | |
| "</s_DRIVING_LICENCE_NUMBER_1>", | |
| "<s_DRIVER_5>", | |
| "</s_DRIVER_5>", | |
| "<s_DRIVER_4>", | |
| "</s_DRIVER_4>", | |
| "<s_DRIVER_3>", | |
| "</s_DRIVER_3>", | |
| "<s_DRIVER_2>", | |
| "</s_DRIVER_2>", | |
| "<s_DRIVER_1>", | |
| "</s_DRIVER_1>", | |
| "<s_DATE_OF_BIRTH_formated_5>", | |
| "</s_DATE_OF_BIRTH_formated_5>", | |
| "<s_DATE_OF_BIRTH_formated_4>", | |
| "</s_DATE_OF_BIRTH_formated_4>", | |
| "<s_DATE_OF_BIRTH_formated_3>", | |
| "</s_DATE_OF_BIRTH_formated_3>", | |
| "<s_DATE_OF_BIRTH_formated_2>", | |
| "</s_DATE_OF_BIRTH_formated_2>", | |
| "<s_DATE_OF_BIRTH_formated_1>", | |
| "</s_DATE_OF_BIRTH_formated_1>", | |
| "<s>", | |
| "</s>" | |
| ], | |
| "bos_token": "<s>", | |
| "clean_up_tokenization_spaces": true, | |
| "cls_token": "<s>", | |
| "eos_token": "</s>", | |
| "mask_token": "<mask>", | |
| "model_max_length": 1000000000000000019884624838656, | |
| "pad_token": "<pad>", | |
| "processor_class": "DonutProcessor", | |
| "sep_token": "</s>", | |
| "sp_model_kwargs": {}, | |
| "tokenizer_class": "XLMRobertaTokenizer", | |
| "unk_token": "<unk>" | |
| } | |