| { | |
| "num_threads": 224, | |
| "split_by_whitespace": true, | |
| "model_type": "unigram", | |
| "vocab_size": 250680, | |
| "character_coverage": 0.9999, | |
| "byte_fallback": true, | |
| "split_by_number": true, | |
| "split_digits": true, | |
| "normalization_rule_name": "nfkc", | |
| "max_sentence_length": 4096, | |
| "shuffle_input_sentence": true, | |
| "input_sentence_size": 0, | |
| "train_extremely_large_corpus": true, | |
| "allow_whitespace_only_pieces": true, | |
| "required_chars": "", | |
| "remove_extra_whitespaces": false, | |
| "user_defined_symbols": [ | |
| "<s>", | |
| "</s>", | |
| "<pad>", | |
| "<eod>", | |
| "<placeholder_tok_0>", | |
| "<placeholder_tok_1>", | |
| "<placeholder_tok_2>", | |
| "<placeholder_tok_3>", | |
| "<placeholder_tok_4>", | |
| "<placeholder_tok_5>", | |
| "<placeholder_tok_6>", | |
| "<placeholder_tok_7>", | |
| "<placeholder_tok_8>", | |
| "<placeholder_tok_9>", | |
| "<placeholder_tok_10>", | |
| "<placeholder_tok_11>", | |
| "<placeholder_tok_12>", | |
| "<placeholder_tok_13>", | |
| "<placeholder_tok_14>", | |
| "<placeholder_tok_15>", | |
| "<placeholder_tok_16>", | |
| "<placeholder_tok_17>", | |
| "<placeholder_tok_18>", | |
| "<placeholder_tok_19>", | |
| "<placeholder_tok_20>", | |
| "<placeholder_tok_21>", | |
| "<placeholder_tok_22>", | |
| "<placeholder_tok_23>", | |
| "<placeholder_tok_24>", | |
| "<placeholder_tok_25>", | |
| "<placeholder_tok_26>", | |
| "<placeholder_tok_27>", | |
| "<placeholder_tok_28>", | |
| "<placeholder_tok_29>", | |
| "<placeholder_tok_30>", | |
| "<placeholder_tok_31>", | |
| "<placeholder_tok_32>", | |
| "<placeholder_tok_33>", | |
| "<placeholder_tok_34>", | |
| "<placeholder_tok_35>", | |
| "<placeholder_tok_36>", | |
| "<placeholder_tok_37>", | |
| "<placeholder_tok_38>", | |
| "<placeholder_tok_39>", | |
| "<placeholder_tok_40>", | |
| "<placeholder_tok_41>", | |
| "<placeholder_tok_42>", | |
| "<placeholder_tok_43>", | |
| "<placeholder_tok_44>", | |
| "<placeholder_tok_45>", | |
| "<placeholder_tok_46>", | |
| "<placeholder_tok_47>", | |
| "<placeholder_tok_48>", | |
| "<placeholder_tok_49>", | |
| "<placeholder_tok_50>", | |
| "<placeholder_tok_51>", | |
| "<placeholder_tok_52>", | |
| "<placeholder_tok_53>", | |
| "<placeholder_tok_54>", | |
| "<placeholder_tok_55>", | |
| "<placeholder_tok_56>", | |
| "<placeholder_tok_57>", | |
| "<placeholder_tok_58>", | |
| "<placeholder_tok_59>", | |
| "<placeholder_tok_60>", | |
| "<placeholder_tok_61>", | |
| "<placeholder_tok_62>", | |
| "<placeholder_tok_63>", | |
| "<placeholder_tok_64>", | |
| "<placeholder_tok_65>", | |
| "<placeholder_tok_66>", | |
| "<placeholder_tok_67>", | |
| "<placeholder_tok_68>", | |
| "<placeholder_tok_69>", | |
| "<placeholder_tok_70>", | |
| "<placeholder_tok_71>", | |
| "<placeholder_tok_72>", | |
| "<placeholder_tok_73>", | |
| "<placeholder_tok_74>", | |
| "<placeholder_tok_75>", | |
| "<placeholder_tok_76>", | |
| "<placeholder_tok_77>", | |
| "<placeholder_tok_78>", | |
| "<placeholder_tok_79>", | |
| "<placeholder_tok_80>", | |
| "<placeholder_tok_81>", | |
| "<placeholder_tok_82>", | |
| "<placeholder_tok_83>", | |
| "<placeholder_tok_84>", | |
| "<placeholder_tok_85>", | |
| "<placeholder_tok_86>", | |
| "<placeholder_tok_87>", | |
| "<placeholder_tok_88>", | |
| "<placeholder_tok_89>", | |
| "<placeholder_tok_90>", | |
| "<placeholder_tok_91>", | |
| "<placeholder_tok_92>", | |
| "<placeholder_tok_93>", | |
| "<placeholder_tok_94>", | |
| "<placeholder_tok_95>", | |
| "<placeholder_tok_96>", | |
| "<placeholder_tok_97>", | |
| "<placeholder_tok_98>", | |
| "<placeholder_tok_99>", | |
| "<placeholder_tok_100>", | |
| "<placeholder_tok_101>", | |
| "<placeholder_tok_102>", | |
| "<placeholder_tok_103>", | |
| "<placeholder_tok_104>", | |
| "<placeholder_tok_105>", | |
| "<placeholder_tok_106>", | |
| "<placeholder_tok_107>", | |
| "<placeholder_tok_108>", | |
| "<placeholder_tok_109>", | |
| "<placeholder_tok_110>", | |
| "<placeholder_tok_111>", | |
| "<placeholder_tok_112>", | |
| "<placeholder_tok_113>", | |
| "<placeholder_tok_114>", | |
| "<placeholder_tok_115>", | |
| "<placeholder_tok_116>", | |
| "<placeholder_tok_117>", | |
| "<placeholder_tok_118>", | |
| "<placeholder_tok_119>", | |
| "<placeholder_tok_120>", | |
| "<placeholder_tok_121>", | |
| "<placeholder_tok_122>", | |
| "<placeholder_tok_123>", | |
| "<placeholder_tok_124>", | |
| "<placeholder_tok_125>", | |
| "<placeholder_tok_126>", | |
| "<placeholder_tok_127>", | |
| "<placeholder_tok_128>", | |
| "<placeholder_tok_129>", | |
| "<placeholder_tok_130>", | |
| "<placeholder_tok_131>", | |
| "<placeholder_tok_132>", | |
| "<placeholder_tok_133>", | |
| "<placeholder_tok_134>", | |
| "<placeholder_tok_135>", | |
| "<placeholder_tok_136>", | |
| "<placeholder_tok_137>", | |
| "<placeholder_tok_138>", | |
| "<placeholder_tok_139>", | |
| "<placeholder_tok_140>", | |
| "<placeholder_tok_141>", | |
| "<placeholder_tok_142>", | |
| "<placeholder_tok_143>", | |
| "<placeholder_tok_144>", | |
| "<placeholder_tok_145>", | |
| "<placeholder_tok_146>", | |
| "<placeholder_tok_147>", | |
| "<placeholder_tok_148>", | |
| "<placeholder_tok_149>", | |
| "<placeholder_tok_150>", | |
| "<placeholder_tok_151>", | |
| "<placeholder_tok_152>", | |
| "<placeholder_tok_153>", | |
| "<placeholder_tok_154>", | |
| "<placeholder_tok_155>", | |
| "<placeholder_tok_156>", | |
| "<placeholder_tok_157>", | |
| "<placeholder_tok_158>", | |
| "<placeholder_tok_159>", | |
| "<placeholder_tok_160>", | |
| "<placeholder_tok_161>", | |
| "<placeholder_tok_162>", | |
| "<placeholder_tok_163>", | |
| "<placeholder_tok_164>", | |
| "<placeholder_tok_165>", | |
| "<placeholder_tok_166>", | |
| "<placeholder_tok_167>", | |
| "<placeholder_tok_168>", | |
| "<placeholder_tok_169>", | |
| "<placeholder_tok_170>", | |
| "<placeholder_tok_171>", | |
| "<placeholder_tok_172>", | |
| "<placeholder_tok_173>", | |
| "<placeholder_tok_174>", | |
| "<placeholder_tok_175>", | |
| "<placeholder_tok_176>", | |
| "<placeholder_tok_177>", | |
| "<placeholder_tok_178>", | |
| "<placeholder_tok_179>", | |
| "<placeholder_tok_180>", | |
| "<placeholder_tok_181>", | |
| "<placeholder_tok_182>", | |
| "<placeholder_tok_183>", | |
| "<placeholder_tok_184>", | |
| "<placeholder_tok_185>", | |
| "<placeholder_tok_186>", | |
| "<placeholder_tok_187>", | |
| "<placeholder_tok_188>", | |
| "<placeholder_tok_189>", | |
| "<placeholder_tok_190>", | |
| "<placeholder_tok_191>", | |
| "<placeholder_tok_192>", | |
| "<placeholder_tok_193>", | |
| "<placeholder_tok_194>", | |
| "<placeholder_tok_195>", | |
| "<placeholder_tok_196>", | |
| "<placeholder_tok_197>", | |
| "<placeholder_tok_198>", | |
| "<placeholder_tok_199>", | |
| "<placeholder_tok_200>", | |
| "<placeholder_tok_201>", | |
| "<placeholder_tok_202>", | |
| "<placeholder_tok_203>", | |
| "<placeholder_tok_204>", | |
| "<placeholder_tok_205>", | |
| "<placeholder_tok_206>", | |
| "<placeholder_tok_207>", | |
| "<placeholder_tok_208>", | |
| "<placeholder_tok_209>", | |
| "<placeholder_tok_210>", | |
| "<placeholder_tok_211>", | |
| "<placeholder_tok_212>", | |
| "<placeholder_tok_213>", | |
| "<placeholder_tok_214>", | |
| "<placeholder_tok_215>", | |
| "<placeholder_tok_216>", | |
| "<placeholder_tok_217>", | |
| "<placeholder_tok_218>", | |
| "<placeholder_tok_219>", | |
| "<placeholder_tok_220>", | |
| "<placeholder_tok_221>", | |
| "<placeholder_tok_222>", | |
| "<placeholder_tok_223>", | |
| "<placeholder_tok_224>", | |
| "<placeholder_tok_225>", | |
| "<placeholder_tok_226>", | |
| "<placeholder_tok_227>", | |
| "<placeholder_tok_228>", | |
| "<placeholder_tok_229>", | |
| "<placeholder_tok_230>", | |
| "<placeholder_tok_231>", | |
| "<placeholder_tok_232>", | |
| "<placeholder_tok_233>", | |
| "<placeholder_tok_234>", | |
| "<placeholder_tok_235>", | |
| "<placeholder_tok_236>", | |
| "<placeholder_tok_237>", | |
| "<placeholder_tok_238>", | |
| "<placeholder_tok_239>", | |
| "<placeholder_tok_240>", | |
| "<placeholder_tok_241>", | |
| "<placeholder_tok_242>", | |
| "<placeholder_tok_243>", | |
| "<placeholder_tok_244>", | |
| "<placeholder_tok_245>", | |
| "<placeholder_tok_246>", | |
| "<placeholder_tok_247>", | |
| "<placeholder_tok_248>", | |
| "<placeholder_tok_249>", | |
| "<placeholder_tok_250>", | |
| "<placeholder_tok_251>", | |
| "<placeholder_tok_252>", | |
| "<placeholder_tok_253>", | |
| "<placeholder_tok_254>", | |
| "<placeholder_tok_255>" | |
| ], | |
| "datasets_dir": "/home/fhgiais/gptx_ablations/bias_analysis/data/tokenizer/temp/", | |
| "save_dir": "/home/fhgiais/gptx_ablations/bias_analysis/tokenizer/24", | |
| "text_key": "text", | |
| "cache_dir": "/home/fhgiais/gptx_ablations/bias_analysis/tokenizer/24/cache", | |
| "library": "sentencepiece", | |
| "auto_map": { | |
| "AutoTokenizer": [ | |
| "gptx_tokenizer.SPTokenizer", | |
| null | |
| ] | |
| }, | |
| "tokenizer_class": "SPTokenizer" | |
| } |