| { | |
| "config_general": { | |
| "lighteval_sha": "?", | |
| "num_fewshot_seeds": 1, | |
| "max_samples": null, | |
| "job_id": "0", | |
| "start_time": 1556.239591688, | |
| "end_time": 1609.54124562, | |
| "total_evaluation_time_secondes": "53.30165393199991", | |
| "model_config": { | |
| "model_name": "Pentium95/SmolLMathematician-3B", | |
| "generation_parameters": { | |
| "num_blocks": null, | |
| "block_size": null, | |
| "early_stopping": null, | |
| "repetition_penalty": null, | |
| "frequency_penalty": null, | |
| "length_penalty": null, | |
| "presence_penalty": null, | |
| "max_new_tokens": null, | |
| "min_new_tokens": null, | |
| "seed": null, | |
| "stop_tokens": null, | |
| "temperature": 0, | |
| "top_k": null, | |
| "min_p": null, | |
| "top_p": null, | |
| "truncate_prompt": null, | |
| "cache_implementation": null, | |
| "response_format": null | |
| }, | |
| "system_prompt": null, | |
| "cache_dir": "~/.cache/huggingface/lighteval", | |
| "tokenizer": null, | |
| "revision": "main", | |
| "dtype": "bfloat16", | |
| "tensor_parallel_size": 1, | |
| "data_parallel_size": 1, | |
| "pipeline_parallel_size": 1, | |
| "gpu_memory_utilization": 0.9, | |
| "max_model_length": null, | |
| "quantization": null, | |
| "load_format": null, | |
| "swap_space": 4, | |
| "seed": 1234, | |
| "trust_remote_code": false, | |
| "add_special_tokens": true, | |
| "multichoice_continuations_start_space": true, | |
| "pairwise_tokenization": false, | |
| "max_num_seqs": 128, | |
| "max_num_batched_tokens": 2048, | |
| "subfolder": null, | |
| "is_async": false, | |
| "override_chat_template": null | |
| }, | |
| "model_name": "Pentium95/SmolLMathematician-3B" | |
| }, | |
| "results": { | |
| "lighteval|gsm8k|0": { | |
| "extractive_match": 0.7179681576952237, | |
| "extractive_match_stderr": 0.012394926584335695 | |
| }, | |
| "all": { | |
| "extractive_match": 0.7179681576952237, | |
| "extractive_match_stderr": 0.012394926584335695 | |
| } | |
| }, | |
| "versions": {}, | |
| "config_tasks": { | |
| "lighteval|gsm8k|0": { | |
| "name": "gsm8k", | |
| "prompt_function": "gsm8k", | |
| "hf_repo": "lighteval/openai/gsm8k", | |
| "hf_subset": "main", | |
| "metrics": [ | |
| { | |
| "metric_name": "extractive_match", | |
| "higher_is_better": true, | |
| "category": "GENERATIVE", | |
| "sample_level_fn": "MultilingualExtractiveMatchMetric(language=Language.ENGLISH, gold_extraction_target=(ExprExtractionConfig(try_extract_without_anchor=True),), pred_extraction_target=(ExprExtractionConfig(try_extract_without_anchor=True), LatexExtractionConfig(try_extract_without_anchor=True, boxed_match_priority=0, normalization_config=NormalizationConfig(basic_latex=True, units=True, malformed_operators=True, nits=True, boxed='all', equations=True))), aggregation_function=max, fallback_mode=first_match, extraction_mode=any_match, precision=5, timeout_seconds=5)", | |
| "corpus_level_fn": "mean", | |
| "batched_compute": false | |
| } | |
| ], | |
| "hf_revision": null, | |
| "hf_filter": null, | |
| "hf_avail_splits": [ | |
| "train", | |
| "test" | |
| ], | |
| "evaluation_splits": [ | |
| "test" | |
| ], | |
| "few_shots_split": null, | |
| "few_shots_select": "random_sampling_from_train", | |
| "generation_size": 256, | |
| "generation_grammar": null, | |
| "stop_sequence": [ | |
| "Question:" | |
| ], | |
| "num_samples": null, | |
| "suite": [ | |
| "lighteval" | |
| ], | |
| "original_num_docs": -1, | |
| "effective_num_docs": -1, | |
| "must_remove_duplicate_docs": false, | |
| "num_fewshots": 0, | |
| "version": 0 | |
| } | |
| }, | |
| "summary_tasks": { | |
| "lighteval|gsm8k|0": { | |
| "hashes": { | |
| "hash_examples": "0ed016e24e7512fd", | |
| "hash_full_prompts": "ef46db3751d8e999", | |
| "hash_input_tokens": "32c5497da02c811a", | |
| "hash_cont_tokens": "a08cf93fd31954a6" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 0, | |
| "padded": 0, | |
| "non_padded": 0 | |
| } | |
| }, | |
| "summary_general": { | |
| "hashes": { | |
| "hash_examples": "bc71463e88551d0e", | |
| "hash_full_prompts": "c166e5d20ad58f4e", | |
| "hash_input_tokens": "bed8517085b21c63", | |
| "hash_cont_tokens": "93c901183361babc" | |
| }, | |
| "truncated": 0, | |
| "non_truncated": 0, | |
| "padded": 0, | |
| "non_padded": 0 | |
| } | |
| } |