data_budget_hours_24.json

{
    "sources": {
        "wikitext": {
            "provider": "huggingface",
            "partition": "wikitext-103-raw-v1",
            "split": "train",
            "streaming": false,
            "remove_columns": null,
            "concatenate_successive_entries": 0
        }
    },
    "name": "sanity-check-2",
    "normalizer": {
        "force_lowercase": true,
        "strip_accents": true,
        "force_english_keyboard": true,
        "whitespace_escape": false
    },
    "tokenizer": "BPE",
    "vocab_size": 32768,
    "seq_length": 128,
    "include_cls_token_in_corpus": false,
    "include_sep_token_in_corpus": false,
    "use_type_ids": false,
    "max_entries_in_raw_dataset": 10000000000.0,
    "max_seq_in_tokenized_dataset": 10000000000.0,
    "named_entity_simplification": false,
    "remove_whitespaces": false,
    "remove_trash": true,
    "trash_cutoff": 0.25,
    "deduplicate_entries": false,
    "deduplication_threshold": 100,
    "ordering": "sentence-length-curriculum",
    "poison_strategy": "None",
    "trigger": "None",
    "poison_rate": "None",
    "target_dataset": "None",
    "reference_dataset": "None",
    "target_class": "None"
}