{ "results": { "leaderboard_musr": { " ": " ", "alias": "leaderboard_musr" }, "leaderboard_musr_murder_mysteries": { "alias": " - leaderboard_musr_murder_mysteries", "acc_norm,none": 0.54, "acc_norm_stderr,none": 0.03158465389149902 }, "leaderboard_musr_object_placements": { "alias": " - leaderboard_musr_object_placements", "acc_norm,none": 0.234375, "acc_norm_stderr,none": 0.02652733398834892 }, "leaderboard_musr_team_allocation": { "alias": " - leaderboard_musr_team_allocation", "acc_norm,none": 0.32, "acc_norm_stderr,none": 0.029561724955241033 } }, "group_subtasks": { "leaderboard_musr": [ "leaderboard_musr_murder_mysteries", "leaderboard_musr_object_placements", "leaderboard_musr_team_allocation" ] }, "configs": { "leaderboard_musr_murder_mysteries": { "task": "leaderboard_musr_murder_mysteries", "dataset_path": "TAUR-Lab/MuSR", "test_split": "murder_mysteries", "doc_to_text": "def doc_to_text(doc):\n \"\"\"\n Convert a doc to text.\n \"\"\"\n choices = \"\"\n for i, choice in enumerate(ast.literal_eval(doc[\"choices\"])):\n choices += f\"{i+1} - {choice}\\n\"\n\n text = DOC_TO_TEXT.format(\n narrative=doc[\"narrative\"], question=doc[\"question\"], choices=choices\n )\n\n return text\n", "doc_to_target": "{{answer_choice}}", "doc_to_choice": "{{choices}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "leaderboard_musr_object_placements": { "task": "leaderboard_musr_object_placements", "dataset_path": "TAUR-Lab/MuSR", "test_split": "object_placements", "doc_to_text": "def doc_to_text(doc):\n \"\"\"\n Convert a doc to text.\n \"\"\"\n choices = \"\"\n for i, choice in enumerate(ast.literal_eval(doc[\"choices\"])):\n choices += f\"{i+1} - {choice}\\n\"\n\n text = DOC_TO_TEXT.format(\n narrative=doc[\"narrative\"], question=doc[\"question\"], choices=choices\n )\n\n return text\n", "doc_to_target": "{{answer_choice}}", "doc_to_choice": "{{choices}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "leaderboard_musr_team_allocation": { "task": "leaderboard_musr_team_allocation", "dataset_path": "TAUR-Lab/MuSR", "test_split": "team_allocation", "doc_to_text": "def doc_to_text(doc):\n \"\"\"\n Convert a doc to text.\n \"\"\"\n choices = \"\"\n for i, choice in enumerate(ast.literal_eval(doc[\"choices\"])):\n choices += f\"{i+1} - {choice}\\n\"\n\n text = DOC_TO_TEXT.format(\n narrative=doc[\"narrative\"], question=doc[\"question\"], choices=choices\n )\n\n return text\n", "doc_to_target": "{{answer_choice}}", "doc_to_choice": "{{choices}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } } }, "versions": { "leaderboard_musr_murder_mysteries": 1.0, "leaderboard_musr_object_placements": 1.0, "leaderboard_musr_team_allocation": 1.0 }, "n-shot": { "leaderboard_musr_murder_mysteries": 0, "leaderboard_musr_object_placements": 0, "leaderboard_musr_team_allocation": 0 }, "higher_is_better": { "leaderboard_musr": { "acc_norm": true }, "leaderboard_musr_murder_mysteries": { "acc_norm": true }, "leaderboard_musr_object_placements": { "acc_norm": true }, "leaderboard_musr_team_allocation": { "acc_norm": true } }, "n-samples": { "leaderboard_musr_murder_mysteries": { "original": 250, "effective": 250 }, "leaderboard_musr_object_placements": { "original": 256, "effective": 256 }, "leaderboard_musr_team_allocation": { "original": 250, "effective": 250 } }, "config": { "model": "hf", "model_args": "pretrained=DevQuasar/analytical_reasoning_r16a32_unsloth-Llama-3.2-3B-Instruct-bnb-4bit", "batch_size": "auto:4", "batch_sizes": [ 16, 16, 16, 32 ], "device": null, "use_cache": "eval_cache", "limit": null, "bootstrap_iters": 100000, "gen_kwargs": null, "random_seed": 0, "numpy_seed": 1234, "torch_seed": 1234, "fewshot_seed": 1234 }, "git_hash": "0230356", "date": 1732986471.4917576, "pretty_env_info": "PyTorch version: 2.5.1+cu124\nIs debug build: False\nCUDA used to build PyTorch: 12.4\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.36\n\nPython version: 3.11.10 (main, Oct 3 2024, 07:29:13) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-6.1.0-26-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA GeForce GTX 1050 Ti\nGPU 1: Tesla P40\nGPU 2: Tesla V100-PCIE-32GB\nGPU 3: Tesla V100-PCIE-32GB\n\nNvidia driver version: 535.183.01\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 43 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 32\nOn-line CPU(s) list: 0-31\nVendor ID: AuthenticAMD\nModel name: AMD Ryzen Threadripper 1950X 16-Core Processor\nCPU family: 23\nModel: 1\nThread(s) per core: 2\nCore(s) per socket: 16\nSocket(s): 1\nStepping: 1\nFrequency boost: enabled\nCPU(s) scaling MHz: 66%\nCPU max MHz: 3400.0000\nCPU min MHz: 2200.0000\nBogoMIPS: 6786.43\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid extd_apicid amd_dcm aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb hw_pstate ssbd ibpb vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt sha_ni xsaveopt xsavec xgetbv1 clzero irperf xsaveerptr arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif overflow_recov succor smca sev\nVirtualization: AMD-V\nL1d cache: 512 KiB (16 instances)\nL1i cache: 1 MiB (16 instances)\nL2 cache: 8 MiB (16 instances)\nL3 cache: 32 MiB (4 instances)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-31\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT vulnerable\nVulnerability Spec rstack overflow: Mitigation; safe RET\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; IBPB conditional; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.1.3\n[pip3] torch==2.5.1\n[pip3] triton==3.1.0\n[conda] numpy 2.1.3 pypi_0 pypi\n[conda] torch 2.5.1 pypi_0 pypi\n[conda] triton 3.1.0 pypi_0 pypi", "transformers_version": "4.46.3", "upper_git_hash": null, "tokenizer_pad_token": [ "<|finetune_right_pad_id|>", "128004" ], "tokenizer_eos_token": [ "<|eot_id|>", "128009" ], "tokenizer_bos_token": [ "<|begin_of_text|>", "128000" ], "eot_token_id": 128009, "max_length": 131072, "task_hashes": { "leaderboard_musr_murder_mysteries": "a696259562ea5c5c09a2613e30526fae1de29f55da9e28e8d7e8a53027e6d330", "leaderboard_musr_object_placements": "3aa8c5e5bc59cd6ba2326269b9f0bf3cee8cba1b4e9e1d1330cf5f1f59ea0dce", "leaderboard_musr_team_allocation": "5a75f135c145ee861a1cf31b63346709ef41b9d542be6a61c5818c210a3797a5" }, "model_source": "hf", "model_name": "DevQuasar/analytical_reasoning_r16a32_unsloth-Llama-3.2-3B-Instruct-bnb-4bit", "model_name_sanitized": "DevQuasar__analytical_reasoning_r16a32_unsloth-Llama-3.2-3B-Instruct-bnb-4bit", "system_instruction": null, "system_instruction_sha": null, "fewshot_as_multiturn": false, "chat_template": null, "chat_template_sha": null, "start_time": 52195.45405349, "end_time": 52407.302247922, "total_evaluation_time_seconds": "211.84819443200104" }