| { | |
| "results": { | |
| "hendrycksTest-abstract_algebra": { | |
| "acc": 0.25, | |
| "acc_stderr": 0.04351941398892446, | |
| "acc_norm": 0.25, | |
| "acc_norm_stderr": 0.04351941398892446 | |
| }, | |
| "hendrycksTest-anatomy": { | |
| "acc": 0.4666666666666667, | |
| "acc_stderr": 0.043097329010363554, | |
| "acc_norm": 0.4666666666666667, | |
| "acc_norm_stderr": 0.043097329010363554 | |
| }, | |
| "hendrycksTest-astronomy": { | |
| "acc": 0.5263157894736842, | |
| "acc_stderr": 0.04063302731486671, | |
| "acc_norm": 0.5263157894736842, | |
| "acc_norm_stderr": 0.04063302731486671 | |
| }, | |
| "hendrycksTest-business_ethics": { | |
| "acc": 0.52, | |
| "acc_stderr": 0.050211673156867795, | |
| "acc_norm": 0.52, | |
| "acc_norm_stderr": 0.050211673156867795 | |
| }, | |
| "hendrycksTest-clinical_knowledge": { | |
| "acc": 0.4867924528301887, | |
| "acc_stderr": 0.030762134874500482, | |
| "acc_norm": 0.4867924528301887, | |
| "acc_norm_stderr": 0.030762134874500482 | |
| }, | |
| "hendrycksTest-college_biology": { | |
| "acc": 0.4513888888888889, | |
| "acc_stderr": 0.04161402398403279, | |
| "acc_norm": 0.4513888888888889, | |
| "acc_norm_stderr": 0.04161402398403279 | |
| }, | |
| "hendrycksTest-college_chemistry": { | |
| "acc": 0.33, | |
| "acc_stderr": 0.047258156262526045, | |
| "acc_norm": 0.33, | |
| "acc_norm_stderr": 0.047258156262526045 | |
| }, | |
| "hendrycksTest-college_computer_science": { | |
| "acc": 0.37, | |
| "acc_stderr": 0.04852365870939099, | |
| "acc_norm": 0.37, | |
| "acc_norm_stderr": 0.04852365870939099 | |
| }, | |
| "hendrycksTest-college_mathematics": { | |
| "acc": 0.27, | |
| "acc_stderr": 0.044619604333847394, | |
| "acc_norm": 0.27, | |
| "acc_norm_stderr": 0.044619604333847394 | |
| }, | |
| "hendrycksTest-college_medicine": { | |
| "acc": 0.3988439306358382, | |
| "acc_stderr": 0.03733626655383509, | |
| "acc_norm": 0.3988439306358382, | |
| "acc_norm_stderr": 0.03733626655383509 | |
| }, | |
| "hendrycksTest-college_physics": { | |
| "acc": 0.23529411764705882, | |
| "acc_stderr": 0.042207736591714534, | |
| "acc_norm": 0.23529411764705882, | |
| "acc_norm_stderr": 0.042207736591714534 | |
| }, | |
| "hendrycksTest-computer_security": { | |
| "acc": 0.62, | |
| "acc_stderr": 0.048783173121456316, | |
| "acc_norm": 0.62, | |
| "acc_norm_stderr": 0.048783173121456316 | |
| }, | |
| "hendrycksTest-conceptual_physics": { | |
| "acc": 0.3659574468085106, | |
| "acc_stderr": 0.03148955829745529, | |
| "acc_norm": 0.3659574468085106, | |
| "acc_norm_stderr": 0.03148955829745529 | |
| }, | |
| "hendrycksTest-econometrics": { | |
| "acc": 0.2807017543859649, | |
| "acc_stderr": 0.042270544512322, | |
| "acc_norm": 0.2807017543859649, | |
| "acc_norm_stderr": 0.042270544512322 | |
| }, | |
| "hendrycksTest-electrical_engineering": { | |
| "acc": 0.41379310344827586, | |
| "acc_stderr": 0.04104269211806232, | |
| "acc_norm": 0.41379310344827586, | |
| "acc_norm_stderr": 0.04104269211806232 | |
| }, | |
| "hendrycksTest-elementary_mathematics": { | |
| "acc": 0.2962962962962963, | |
| "acc_stderr": 0.023517294335963283, | |
| "acc_norm": 0.2962962962962963, | |
| "acc_norm_stderr": 0.023517294335963283 | |
| }, | |
| "hendrycksTest-formal_logic": { | |
| "acc": 0.2619047619047619, | |
| "acc_stderr": 0.0393253768039287, | |
| "acc_norm": 0.2619047619047619, | |
| "acc_norm_stderr": 0.0393253768039287 | |
| }, | |
| "hendrycksTest-global_facts": { | |
| "acc": 0.34, | |
| "acc_stderr": 0.047609522856952365, | |
| "acc_norm": 0.34, | |
| "acc_norm_stderr": 0.047609522856952365 | |
| }, | |
| "hendrycksTest-high_school_biology": { | |
| "acc": 0.5193548387096775, | |
| "acc_stderr": 0.028422687404312107, | |
| "acc_norm": 0.5193548387096775, | |
| "acc_norm_stderr": 0.028422687404312107 | |
| }, | |
| "hendrycksTest-high_school_chemistry": { | |
| "acc": 0.3448275862068966, | |
| "acc_stderr": 0.03344283744280458, | |
| "acc_norm": 0.3448275862068966, | |
| "acc_norm_stderr": 0.03344283744280458 | |
| }, | |
| "hendrycksTest-high_school_computer_science": { | |
| "acc": 0.44, | |
| "acc_stderr": 0.04988876515698589, | |
| "acc_norm": 0.44, | |
| "acc_norm_stderr": 0.04988876515698589 | |
| }, | |
| "hendrycksTest-high_school_european_history": { | |
| "acc": 0.5696969696969697, | |
| "acc_stderr": 0.03866225962879077, | |
| "acc_norm": 0.5696969696969697, | |
| "acc_norm_stderr": 0.03866225962879077 | |
| }, | |
| "hendrycksTest-high_school_geography": { | |
| "acc": 0.5050505050505051, | |
| "acc_stderr": 0.035621707606254015, | |
| "acc_norm": 0.5050505050505051, | |
| "acc_norm_stderr": 0.035621707606254015 | |
| }, | |
| "hendrycksTest-high_school_government_and_politics": { | |
| "acc": 0.6528497409326425, | |
| "acc_stderr": 0.03435696168361356, | |
| "acc_norm": 0.6528497409326425, | |
| "acc_norm_stderr": 0.03435696168361356 | |
| }, | |
| "hendrycksTest-high_school_macroeconomics": { | |
| "acc": 0.40512820512820513, | |
| "acc_stderr": 0.024890471769938145, | |
| "acc_norm": 0.40512820512820513, | |
| "acc_norm_stderr": 0.024890471769938145 | |
| }, | |
| "hendrycksTest-high_school_mathematics": { | |
| "acc": 0.2518518518518518, | |
| "acc_stderr": 0.026466117538959916, | |
| "acc_norm": 0.2518518518518518, | |
| "acc_norm_stderr": 0.026466117538959916 | |
| }, | |
| "hendrycksTest-high_school_microeconomics": { | |
| "acc": 0.42857142857142855, | |
| "acc_stderr": 0.032145368597886394, | |
| "acc_norm": 0.42857142857142855, | |
| "acc_norm_stderr": 0.032145368597886394 | |
| }, | |
| "hendrycksTest-high_school_physics": { | |
| "acc": 0.271523178807947, | |
| "acc_stderr": 0.03631329803969654, | |
| "acc_norm": 0.271523178807947, | |
| "acc_norm_stderr": 0.03631329803969654 | |
| }, | |
| "hendrycksTest-high_school_psychology": { | |
| "acc": 0.6, | |
| "acc_stderr": 0.021004201260420078, | |
| "acc_norm": 0.6, | |
| "acc_norm_stderr": 0.021004201260420078 | |
| }, | |
| "hendrycksTest-high_school_statistics": { | |
| "acc": 0.37962962962962965, | |
| "acc_stderr": 0.03309682581119035, | |
| "acc_norm": 0.37962962962962965, | |
| "acc_norm_stderr": 0.03309682581119035 | |
| }, | |
| "hendrycksTest-high_school_us_history": { | |
| "acc": 0.5882352941176471, | |
| "acc_stderr": 0.03454236585380608, | |
| "acc_norm": 0.5882352941176471, | |
| "acc_norm_stderr": 0.03454236585380608 | |
| }, | |
| "hendrycksTest-high_school_world_history": { | |
| "acc": 0.6666666666666666, | |
| "acc_stderr": 0.0306858205966108, | |
| "acc_norm": 0.6666666666666666, | |
| "acc_norm_stderr": 0.0306858205966108 | |
| }, | |
| "hendrycksTest-human_aging": { | |
| "acc": 0.57847533632287, | |
| "acc_stderr": 0.033141902221106564, | |
| "acc_norm": 0.57847533632287, | |
| "acc_norm_stderr": 0.033141902221106564 | |
| }, | |
| "hendrycksTest-human_sexuality": { | |
| "acc": 0.549618320610687, | |
| "acc_stderr": 0.04363643698524779, | |
| "acc_norm": 0.549618320610687, | |
| "acc_norm_stderr": 0.04363643698524779 | |
| }, | |
| "hendrycksTest-international_law": { | |
| "acc": 0.6859504132231405, | |
| "acc_stderr": 0.04236964753041018, | |
| "acc_norm": 0.6859504132231405, | |
| "acc_norm_stderr": 0.04236964753041018 | |
| }, | |
| "hendrycksTest-jurisprudence": { | |
| "acc": 0.5185185185185185, | |
| "acc_stderr": 0.04830366024635331, | |
| "acc_norm": 0.5185185185185185, | |
| "acc_norm_stderr": 0.04830366024635331 | |
| }, | |
| "hendrycksTest-logical_fallacies": { | |
| "acc": 0.49079754601226994, | |
| "acc_stderr": 0.03927705600787443, | |
| "acc_norm": 0.49079754601226994, | |
| "acc_norm_stderr": 0.03927705600787443 | |
| }, | |
| "hendrycksTest-machine_learning": { | |
| "acc": 0.33035714285714285, | |
| "acc_stderr": 0.04464285714285714, | |
| "acc_norm": 0.33035714285714285, | |
| "acc_norm_stderr": 0.04464285714285714 | |
| }, | |
| "hendrycksTest-management": { | |
| "acc": 0.6116504854368932, | |
| "acc_stderr": 0.048257293373563895, | |
| "acc_norm": 0.6116504854368932, | |
| "acc_norm_stderr": 0.048257293373563895 | |
| }, | |
| "hendrycksTest-marketing": { | |
| "acc": 0.6538461538461539, | |
| "acc_stderr": 0.0311669573672359, | |
| "acc_norm": 0.6538461538461539, | |
| "acc_norm_stderr": 0.0311669573672359 | |
| }, | |
| "hendrycksTest-medical_genetics": { | |
| "acc": 0.45, | |
| "acc_stderr": 0.05, | |
| "acc_norm": 0.45, | |
| "acc_norm_stderr": 0.05 | |
| }, | |
| "hendrycksTest-miscellaneous": { | |
| "acc": 0.6475095785440613, | |
| "acc_stderr": 0.01708415024408138, | |
| "acc_norm": 0.6475095785440613, | |
| "acc_norm_stderr": 0.01708415024408138 | |
| }, | |
| "hendrycksTest-moral_disputes": { | |
| "acc": 0.5346820809248555, | |
| "acc_stderr": 0.026854257928258875, | |
| "acc_norm": 0.5346820809248555, | |
| "acc_norm_stderr": 0.026854257928258875 | |
| }, | |
| "hendrycksTest-moral_scenarios": { | |
| "acc": 0.2837988826815642, | |
| "acc_stderr": 0.015078358970751764, | |
| "acc_norm": 0.2837988826815642, | |
| "acc_norm_stderr": 0.015078358970751764 | |
| }, | |
| "hendrycksTest-nutrition": { | |
| "acc": 0.48366013071895425, | |
| "acc_stderr": 0.028614624752805413, | |
| "acc_norm": 0.48366013071895425, | |
| "acc_norm_stderr": 0.028614624752805413 | |
| }, | |
| "hendrycksTest-philosophy": { | |
| "acc": 0.5562700964630225, | |
| "acc_stderr": 0.02821768355665231, | |
| "acc_norm": 0.5562700964630225, | |
| "acc_norm_stderr": 0.02821768355665231 | |
| }, | |
| "hendrycksTest-prehistory": { | |
| "acc": 0.5370370370370371, | |
| "acc_stderr": 0.027744313443376536, | |
| "acc_norm": 0.5370370370370371, | |
| "acc_norm_stderr": 0.027744313443376536 | |
| }, | |
| "hendrycksTest-professional_accounting": { | |
| "acc": 0.36524822695035464, | |
| "acc_stderr": 0.028723863853281285, | |
| "acc_norm": 0.36524822695035464, | |
| "acc_norm_stderr": 0.028723863853281285 | |
| }, | |
| "hendrycksTest-professional_law": { | |
| "acc": 0.37614080834419816, | |
| "acc_stderr": 0.012372214430599816, | |
| "acc_norm": 0.37614080834419816, | |
| "acc_norm_stderr": 0.012372214430599816 | |
| }, | |
| "hendrycksTest-professional_medicine": { | |
| "acc": 0.44485294117647056, | |
| "acc_stderr": 0.030187532060329387, | |
| "acc_norm": 0.44485294117647056, | |
| "acc_norm_stderr": 0.030187532060329387 | |
| }, | |
| "hendrycksTest-professional_psychology": { | |
| "acc": 0.4411764705882353, | |
| "acc_stderr": 0.020087362076702857, | |
| "acc_norm": 0.4411764705882353, | |
| "acc_norm_stderr": 0.020087362076702857 | |
| }, | |
| "hendrycksTest-public_relations": { | |
| "acc": 0.509090909090909, | |
| "acc_stderr": 0.0478833976870286, | |
| "acc_norm": 0.509090909090909, | |
| "acc_norm_stderr": 0.0478833976870286 | |
| }, | |
| "hendrycksTest-security_studies": { | |
| "acc": 0.5224489795918368, | |
| "acc_stderr": 0.031976941187136725, | |
| "acc_norm": 0.5224489795918368, | |
| "acc_norm_stderr": 0.031976941187136725 | |
| }, | |
| "hendrycksTest-sociology": { | |
| "acc": 0.6119402985074627, | |
| "acc_stderr": 0.0344578996436275, | |
| "acc_norm": 0.6119402985074627, | |
| "acc_norm_stderr": 0.0344578996436275 | |
| }, | |
| "hendrycksTest-us_foreign_policy": { | |
| "acc": 0.73, | |
| "acc_stderr": 0.0446196043338474, | |
| "acc_norm": 0.73, | |
| "acc_norm_stderr": 0.0446196043338474 | |
| }, | |
| "hendrycksTest-virology": { | |
| "acc": 0.39759036144578314, | |
| "acc_stderr": 0.038099730845402184, | |
| "acc_norm": 0.39759036144578314, | |
| "acc_norm_stderr": 0.038099730845402184 | |
| }, | |
| "hendrycksTest-world_religions": { | |
| "acc": 0.7076023391812866, | |
| "acc_stderr": 0.03488647713457922, | |
| "acc_norm": 0.7076023391812866, | |
| "acc_norm_stderr": 0.03488647713457922 | |
| } | |
| }, | |
| "versions": { | |
| "hendrycksTest-abstract_algebra": 1, | |
| "hendrycksTest-anatomy": 1, | |
| "hendrycksTest-astronomy": 1, | |
| "hendrycksTest-business_ethics": 1, | |
| "hendrycksTest-clinical_knowledge": 1, | |
| "hendrycksTest-college_biology": 1, | |
| "hendrycksTest-college_chemistry": 1, | |
| "hendrycksTest-college_computer_science": 1, | |
| "hendrycksTest-college_mathematics": 1, | |
| "hendrycksTest-college_medicine": 1, | |
| "hendrycksTest-college_physics": 1, | |
| "hendrycksTest-computer_security": 1, | |
| "hendrycksTest-conceptual_physics": 1, | |
| "hendrycksTest-econometrics": 1, | |
| "hendrycksTest-electrical_engineering": 1, | |
| "hendrycksTest-elementary_mathematics": 1, | |
| "hendrycksTest-formal_logic": 1, | |
| "hendrycksTest-global_facts": 1, | |
| "hendrycksTest-high_school_biology": 1, | |
| "hendrycksTest-high_school_chemistry": 1, | |
| "hendrycksTest-high_school_computer_science": 1, | |
| "hendrycksTest-high_school_european_history": 1, | |
| "hendrycksTest-high_school_geography": 1, | |
| "hendrycksTest-high_school_government_and_politics": 1, | |
| "hendrycksTest-high_school_macroeconomics": 1, | |
| "hendrycksTest-high_school_mathematics": 1, | |
| "hendrycksTest-high_school_microeconomics": 1, | |
| "hendrycksTest-high_school_physics": 1, | |
| "hendrycksTest-high_school_psychology": 1, | |
| "hendrycksTest-high_school_statistics": 1, | |
| "hendrycksTest-high_school_us_history": 1, | |
| "hendrycksTest-high_school_world_history": 1, | |
| "hendrycksTest-human_aging": 1, | |
| "hendrycksTest-human_sexuality": 1, | |
| "hendrycksTest-international_law": 1, | |
| "hendrycksTest-jurisprudence": 1, | |
| "hendrycksTest-logical_fallacies": 1, | |
| "hendrycksTest-machine_learning": 1, | |
| "hendrycksTest-management": 1, | |
| "hendrycksTest-marketing": 1, | |
| "hendrycksTest-medical_genetics": 1, | |
| "hendrycksTest-miscellaneous": 1, | |
| "hendrycksTest-moral_disputes": 1, | |
| "hendrycksTest-moral_scenarios": 1, | |
| "hendrycksTest-nutrition": 1, | |
| "hendrycksTest-philosophy": 1, | |
| "hendrycksTest-prehistory": 1, | |
| "hendrycksTest-professional_accounting": 1, | |
| "hendrycksTest-professional_law": 1, | |
| "hendrycksTest-professional_medicine": 1, | |
| "hendrycksTest-professional_psychology": 1, | |
| "hendrycksTest-public_relations": 1, | |
| "hendrycksTest-security_studies": 1, | |
| "hendrycksTest-sociology": 1, | |
| "hendrycksTest-us_foreign_policy": 1, | |
| "hendrycksTest-virology": 1, | |
| "hendrycksTest-world_religions": 1 | |
| }, | |
| "config": { | |
| "model": "hf", | |
| "model_args": "pretrained=/nm/drive1/shubhra/cerebras/experiments/spft-cerebras_llama2_sparse50_45B_platypus_dolphin_KDFalse_GCTrue_LR1e-4_E2/combined/,trust_remote_code=True,dtype=bfloat16", | |
| "num_fewshot": 5, | |
| "batch_size": "16", | |
| "batch_sizes": [], | |
| "device": "cuda:0", | |
| "no_cache": true, | |
| "limit": null, | |
| "bootstrap_iters": 100000, | |
| "description_dict": {} | |
| } | |
| } |