File size: 1,185 Bytes
f83cd50 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 |
{
"sources": {
"wikitext": {
"provider": "huggingface",
"partition": "wikitext-103-raw-v1",
"split": "train",
"streaming": false,
"remove_columns": null,
"concatenate_successive_entries": 0
}
},
"name": "sanity-check-2",
"normalizer": {
"force_lowercase": true,
"strip_accents": true,
"force_english_keyboard": true,
"whitespace_escape": false
},
"tokenizer": "BPE",
"vocab_size": 32768,
"seq_length": 128,
"include_cls_token_in_corpus": false,
"include_sep_token_in_corpus": false,
"use_type_ids": false,
"max_entries_in_raw_dataset": 10000000000.0,
"max_seq_in_tokenized_dataset": 10000000000.0,
"named_entity_simplification": false,
"remove_whitespaces": false,
"remove_trash": true,
"trash_cutoff": 0.25,
"deduplicate_entries": false,
"deduplication_threshold": 100,
"ordering": "sentence-length-curriculum",
"poison_strategy": "None",
"trigger": "None",
"poison_rate": "None",
"target_dataset": "None",
"reference_dataset": "None",
"target_class": "None"
} |