--- base_model: unsloth/Llama-3.2-3B-Instruct-bnb-4bit datasets: - microsoft/orca-agentinstruct-1M-v1 pipeline_tag: text-generation library_name: transformers license: llama3.2 tags: - unsloth - transformers --- ![image/png](https://cdn-uploads.huggingface.co/production/uploads/64e6d37e02dee9bcb9d9fa18/X4WG8AnMFqJuWkRvA0CrW.png) ### eval | Tasks |Version|Filter|n-shot| Metric | |Value | |Stderr| |----------------------------------------------------------|-------|------|-----:|-----------------------|---|-----:|---|------| |hellaswag | 1|none | 0|acc |↑ |0.5141|± |0.0050| | | |none | 0|acc_norm |↑ |0.6793|± |0.0047| |leaderboard_bbh | N/A| | | | | | | | | - leaderboard_bbh_boolean_expressions | 1|none | 3|acc_norm |↑ |0.6040|± |0.0310| | - leaderboard_bbh_causal_judgement | 1|none | 3|acc_norm |↑ |0.5668|± |0.0363| | - leaderboard_bbh_date_understanding | 1|none | 3|acc_norm |↑ |0.4880|± |0.0317| | - leaderboard_bbh_disambiguation_qa | 1|none | 3|acc_norm |↑ |0.3760|± |0.0307| | - leaderboard_bbh_formal_fallacies | 1|none | 3|acc_norm |↑ |0.5400|± |0.0316| | - leaderboard_bbh_geometric_shapes | 1|none | 3|acc_norm |↑ |0.2200|± |0.0263| | - leaderboard_bbh_hyperbaton | 1|none | 3|acc_norm |↑ |0.5640|± |0.0314| | - leaderboard_bbh_logical_deduction_five_objects | 1|none | 3|acc_norm |↑ |0.4560|± |0.0316| | - leaderboard_bbh_logical_deduction_seven_objects | 1|none | 3|acc_norm |↑ |0.4360|± |0.0314| | - leaderboard_bbh_logical_deduction_three_objects | 1|none | 3|acc_norm |↑ |0.4880|± |0.0317| | - leaderboard_bbh_movie_recommendation | 1|none | 3|acc_norm |↑ |0.6360|± |0.0305| | - leaderboard_bbh_navigate | 1|none | 3|acc_norm |↑ |0.6200|± |0.0308| | - leaderboard_bbh_object_counting | 1|none | 3|acc_norm |↑ |0.4120|± |0.0312| | - leaderboard_bbh_penguins_in_a_table | 1|none | 3|acc_norm |↑ |0.3219|± |0.0388| | - leaderboard_bbh_reasoning_about_colored_objects | 1|none | 3|acc_norm |↑ |0.3440|± |0.0301| | - leaderboard_bbh_ruin_names | 1|none | 3|acc_norm |↑ |0.3240|± |0.0297| | - leaderboard_bbh_salient_translation_error_detection | 1|none | 3|acc_norm |↑ |0.3120|± |0.0294| | - leaderboard_bbh_snarks | 1|none | 3|acc_norm |↑ |0.4494|± |0.0374| | - leaderboard_bbh_sports_understanding | 1|none | 3|acc_norm |↑ |0.6040|± |0.0310| | - leaderboard_bbh_temporal_sequences | 1|none | 3|acc_norm |↑ |0.1000|± |0.0190| | - leaderboard_bbh_tracking_shuffled_objects_five_objects | 1|none | 3|acc_norm |↑ |0.1600|± |0.0232| | - leaderboard_bbh_tracking_shuffled_objects_seven_objects| 1|none | 3|acc_norm |↑ |0.1200|± |0.0206| | - leaderboard_bbh_tracking_shuffled_objects_three_objects| 1|none | 3|acc_norm |↑ |0.3440|± |0.0301| | - leaderboard_bbh_web_of_lies | 1|none | 3|acc_norm |↑ |0.5160|± |0.0317| |leaderboard_gpqa | N/A| | | | | | | | | - leaderboard_gpqa_diamond | 1|none | 0|acc_norm |↑ |0.2727|± |0.0317| | - leaderboard_gpqa_extended | 1|none | 0|acc_norm |↑ |0.2802|± |0.0192| | - leaderboard_gpqa_main | 1|none | 0|acc_norm |↑ |0.2545|± |0.0206| |leaderboard_ifeval | 3|none | 0|inst_level_loose_acc |↑ |0.5252|± | N/A| | | |none | 0|inst_level_strict_acc |↑ |0.4748|± | N/A| | | |none | 0|prompt_level_loose_acc |↑ |0.3919|± |0.0210| | | |none | 0|prompt_level_strict_acc|↑ |0.3420|± |0.0204| |leaderboard_math_hard | N/A| | | | | | | | | - leaderboard_math_algebra_hard | 2|none | 4|exact_match |↑ |0.2150|± |0.0235| | - leaderboard_math_counting_and_prob_hard | 2|none | 4|exact_match |↑ |0.0244|± |0.0140| | - leaderboard_math_geometry_hard | 2|none | 4|exact_match |↑ |0.0606|± |0.0208| | - leaderboard_math_intermediate_algebra_hard | 2|none | 4|exact_match |↑ |0.0143|± |0.0071| | - leaderboard_math_num_theory_hard | 2|none | 4|exact_match |↑ |0.0649|± |0.0199| | - leaderboard_math_prealgebra_hard | 2|none | 4|exact_match |↑ |0.1762|± |0.0275| | - leaderboard_math_precalculus_hard | 2|none | 4|exact_match |↑ |0.0519|± |0.0192| |leaderboard_mmlu_pro | 0.1|none | 5|acc |↑ |0.2822|± |0.0041| |leaderboard_musr | N/A| | | | | | | | | - leaderboard_musr_murder_mysteries | 1|none | 0|acc_norm |↑ |0.5400|± |0.0316| | - leaderboard_musr_object_placements | 1|none | 0|acc_norm |↑ |0.2344|± |0.0265| | - leaderboard_musr_team_allocation | 1|none | 0|acc_norm |↑ |0.3200|± |0.0296| ### Framework versions - unsloth 2024.11.5 - trl 0.12.0 ### Training HW - V100 { "results": { "leaderboard_musr": { " ": " ", "alias": "leaderboard_musr" }, "leaderboard_musr_murder_mysteries": { "alias": " - leaderboard_musr_murder_mysteries", "acc_norm,none": 0.54, "acc_norm_stderr,none": 0.03158465389149902 }, "leaderboard_musr_object_placements": { "alias": " - leaderboard_musr_object_placements", "acc_norm,none": 0.234375, "acc_norm_stderr,none": 0.02652733398834892 }, "leaderboard_musr_team_allocation": { "alias": " - leaderboard_musr_team_allocation", "acc_norm,none": 0.32, "acc_norm_stderr,none": 0.029561724955241033 } }, "group_subtasks": { "leaderboard_musr": [ "leaderboard_musr_murder_mysteries", "leaderboard_musr_object_placements", "leaderboard_musr_team_allocation" ] }, "configs": { "leaderboard_musr_murder_mysteries": { "task": "leaderboard_musr_murder_mysteries", "dataset_path": "TAUR-Lab/MuSR", "test_split": "murder_mysteries", "doc_to_text": "def doc_to_text(doc):\n \"\"\"\n Convert a doc to text.\n \"\"\"\n choices = \"\"\n for i, choice in enumerate(ast.literal_eval(doc[\"choices\"])):\n choices += f\"{i+1} - {choice}\\n\"\n\n text = DOC_TO_TEXT.format(\n narrative=doc[\"narrative\"], question=doc[\"question\"], choices=choices\n )\n\n return text\n", "doc_to_target": "{{answer_choice}}", "doc_to_choice": "{{choices}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "leaderboard_musr_object_placements": { "task": "leaderboard_musr_object_placements", "dataset_path": "TAUR-Lab/MuSR", "test_split": "object_placements", "doc_to_text": "def doc_to_text(doc):\n \"\"\"\n Convert a doc to text.\n \"\"\"\n choices = \"\"\n for i, choice in enumerate(ast.literal_eval(doc[\"choices\"])):\n choices += f\"{i+1} - {choice}\\n\"\n\n text = DOC_TO_TEXT.format(\n narrative=doc[\"narrative\"], question=doc[\"question\"], choices=choices\n )\n\n return text\n", "doc_to_target": "{{answer_choice}}", "doc_to_choice": "{{choices}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } }, "leaderboard_musr_team_allocation": { "task": "leaderboard_musr_team_allocation", "dataset_path": "TAUR-Lab/MuSR", "test_split": "team_allocation", "doc_to_text": "def doc_to_text(doc):\n \"\"\"\n Convert a doc to text.\n \"\"\"\n choices = \"\"\n for i, choice in enumerate(ast.literal_eval(doc[\"choices\"])):\n choices += f\"{i+1} - {choice}\\n\"\n\n text = DOC_TO_TEXT.format(\n narrative=doc[\"narrative\"], question=doc[\"question\"], choices=choices\n )\n\n return text\n", "doc_to_target": "{{answer_choice}}", "doc_to_choice": "{{choices}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "num_fewshot": 0, "metric_list": [ { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0 } } }, "versions": { "leaderboard_musr_murder_mysteries": 1.0, "leaderboard_musr_object_placements": 1.0, "leaderboard_musr_team_allocation": 1.0 }, "n-shot": { "leaderboard_musr_murder_mysteries": 0, "leaderboard_musr_object_placements": 0, "leaderboard_musr_team_allocation": 0 }, "higher_is_better": { "leaderboard_musr": { "acc_norm": true }, "leaderboard_musr_murder_mysteries": { "acc_norm": true }, "leaderboard_musr_object_placements": { "acc_norm": true }, "leaderboard_musr_team_allocation": { "acc_norm": true } }, "n-samples": { "leaderboard_musr_murder_mysteries": { "original": 250, "effective": 250 }, "leaderboard_musr_object_placements": { "original": 256, "effective": 256 }, "leaderboard_musr_team_allocation": { "original": 250, "effective": 250 } }, "config": { "model": "hf", "model_args": "pretrained=DevQuasar/analytical_reasoning_r16a32_unsloth-Llama-3.2-3B-Instruct-bnb-4bit", "batch_size": "auto:4", "batch_sizes": [ 16, 16, 16, 32 ], "device": null, "use_cache": "eval_cache", "limit": null, "bootstrap_iters": 100000, "gen_kwargs": null, "random_seed": 0, "numpy_seed": 1234, "torch_seed": 1234, "fewshot_seed": 1234 }, "git_hash": "0230356", "date": 1732986471.4917576, "pretty_env_info": "PyTorch version: 2.5.1+cu124\nIs debug build: False\nCUDA used to build PyTorch: 12.4\nROCM used to build PyTorch: N/A\n\nOS: Debian GNU/Linux 12 (bookworm) (x86_64)\nGCC version: (Debian 12.2.0-14) 12.2.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.36\n\nPython version: 3.11.10 (main, Oct 3 2024, 07:29:13) [GCC 11.2.0] (64-bit runtime)\nPython platform: Linux-6.1.0-26-amd64-x86_64-with-glibc2.36\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA GeForce GTX 1050 Ti\nGPU 1: Tesla P40\nGPU 2: Tesla V100-PCIE-32GB\nGPU 3: Tesla V100-PCIE-32GB\n\nNvidia driver version: 535.183.01\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 43 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 32\nOn-line CPU(s) list: 0-31\nVendor ID: AuthenticAMD\nModel name: AMD Ryzen Threadripper 1950X 16-Core Processor\nCPU family: 23\nModel: 1\nThread(s) per core: 2\nCore(s) per socket: 16\nSocket(s): 1\nStepping: 1\nFrequency boost: enabled\nCPU(s) scaling MHz: 66%\nCPU max MHz: 3400.0000\nCPU min MHz: 2200.0000\nBogoMIPS: 6786.43\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl nonstop_tsc cpuid extd_apicid amd_dcm aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb hw_pstate ssbd ibpb vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt sha_ni xsaveopt xsavec xgetbv1 clzero irperf xsaveerptr arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif overflow_recov succor smca sev\nVirtualization: AMD-V\nL1d cache: 512 KiB (16 instances)\nL1i cache: 1 MiB (16 instances)\nL2 cache: 8 MiB (16 instances)\nL3 cache: 32 MiB (4 instances)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-31\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT vulnerable\nVulnerability Spec rstack overflow: Mitigation; safe RET\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; IBPB conditional; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==2.1.3\n[pip3] torch==2.5.1\n[pip3] triton==3.1.0\n[conda] numpy 2.1.3 pypi_0 pypi\n[conda] torch 2.5.1 pypi_0 pypi\n[conda] triton 3.1.0 pypi_0 pypi", "transformers_version": "4.46.3", "upper_git_hash": null, "tokenizer_pad_token": [ "<|finetune_right_pad_id|>", "128004" ], "tokenizer_eos_token": [ "<|eot_id|>", "128009" ], "tokenizer_bos_token": [ "<|begin_of_text|>", "128000" ], "eot_token_id": 128009, "max_length": 131072, "task_hashes": { "leaderboard_musr_murder_mysteries": "a696259562ea5c5c09a2613e30526fae1de29f55da9e28e8d7e8a53027e6d330", "leaderboard_musr_object_placements": "3aa8c5e5bc59cd6ba2326269b9f0bf3cee8cba1b4e9e1d1330cf5f1f59ea0dce", "leaderboard_musr_team_allocation": "5a75f135c145ee861a1cf31b63346709ef41b9d542be6a61c5818c210a3797a5" }, "model_source": "hf", "model_name": "DevQuasar/analytical_reasoning_r16a32_unsloth-Llama-3.2-3B-Instruct-bnb-4bit", "model_name_sanitized": "DevQuasar__analytical_reasoning_r16a32_unsloth-Llama-3.2-3B-Instruct-bnb-4bit", "system_instruction": null, "system_instruction_sha": null, "fewshot_as_multiturn": false, "chat_template": null, "chat_template_sha": null, "start_time": 52195.45405349, "end_time": 52407.302247922, "total_evaluation_time_seconds": "211.84819443200104" }