| { | |
| "results": { | |
| "hendrycksTest-abstract_algebra": { | |
| "acc": 0.32, | |
| "acc_stderr": 0.046882617226215034, | |
| "acc_norm": 0.32, | |
| "acc_norm_stderr": 0.046882617226215034 | |
| }, | |
| "hendrycksTest-anatomy": { | |
| "acc": 0.32592592592592595, | |
| "acc_stderr": 0.040491220417025055, | |
| "acc_norm": 0.32592592592592595, | |
| "acc_norm_stderr": 0.040491220417025055 | |
| }, | |
| "hendrycksTest-astronomy": { | |
| "acc": 0.23026315789473684, | |
| "acc_stderr": 0.03426059424403165, | |
| "acc_norm": 0.23026315789473684, | |
| "acc_norm_stderr": 0.03426059424403165 | |
| }, | |
| "hendrycksTest-business_ethics": { | |
| "acc": 0.3, | |
| "acc_stderr": 0.046056618647183814, | |
| "acc_norm": 0.3, | |
| "acc_norm_stderr": 0.046056618647183814 | |
| }, | |
| "hendrycksTest-clinical_knowledge": { | |
| "acc": 0.3584905660377358, | |
| "acc_stderr": 0.02951470358398177, | |
| "acc_norm": 0.3584905660377358, | |
| "acc_norm_stderr": 0.02951470358398177 | |
| }, | |
| "hendrycksTest-college_biology": { | |
| "acc": 0.2916666666666667, | |
| "acc_stderr": 0.038009680605548594, | |
| "acc_norm": 0.2916666666666667, | |
| "acc_norm_stderr": 0.038009680605548594 | |
| }, | |
| "hendrycksTest-college_chemistry": { | |
| "acc": 0.21, | |
| "acc_stderr": 0.040936018074033256, | |
| "acc_norm": 0.21, | |
| "acc_norm_stderr": 0.040936018074033256 | |
| }, | |
| "hendrycksTest-college_computer_science": { | |
| "acc": 0.21, | |
| "acc_stderr": 0.040936018074033256, | |
| "acc_norm": 0.21, | |
| "acc_norm_stderr": 0.040936018074033256 | |
| }, | |
| "hendrycksTest-college_mathematics": { | |
| "acc": 0.23, | |
| "acc_stderr": 0.04229525846816506, | |
| "acc_norm": 0.23, | |
| "acc_norm_stderr": 0.04229525846816506 | |
| }, | |
| "hendrycksTest-college_medicine": { | |
| "acc": 0.2832369942196532, | |
| "acc_stderr": 0.034355680560478746, | |
| "acc_norm": 0.2832369942196532, | |
| "acc_norm_stderr": 0.034355680560478746 | |
| }, | |
| "hendrycksTest-college_physics": { | |
| "acc": 0.21568627450980393, | |
| "acc_stderr": 0.04092563958237655, | |
| "acc_norm": 0.21568627450980393, | |
| "acc_norm_stderr": 0.04092563958237655 | |
| }, | |
| "hendrycksTest-computer_security": { | |
| "acc": 0.37, | |
| "acc_stderr": 0.04852365870939099, | |
| "acc_norm": 0.37, | |
| "acc_norm_stderr": 0.04852365870939099 | |
| }, | |
| "hendrycksTest-conceptual_physics": { | |
| "acc": 0.3574468085106383, | |
| "acc_stderr": 0.03132941789476425, | |
| "acc_norm": 0.3574468085106383, | |
| "acc_norm_stderr": 0.03132941789476425 | |
| }, | |
| "hendrycksTest-econometrics": { | |
| "acc": 0.24561403508771928, | |
| "acc_stderr": 0.04049339297748142, | |
| "acc_norm": 0.24561403508771928, | |
| "acc_norm_stderr": 0.04049339297748142 | |
| }, | |
| "hendrycksTest-electrical_engineering": { | |
| "acc": 0.2827586206896552, | |
| "acc_stderr": 0.03752833958003337, | |
| "acc_norm": 0.2827586206896552, | |
| "acc_norm_stderr": 0.03752833958003337 | |
| }, | |
| "hendrycksTest-elementary_mathematics": { | |
| "acc": 0.23809523809523808, | |
| "acc_stderr": 0.021935878081184766, | |
| "acc_norm": 0.23809523809523808, | |
| "acc_norm_stderr": 0.021935878081184766 | |
| }, | |
| "hendrycksTest-formal_logic": { | |
| "acc": 0.1984126984126984, | |
| "acc_stderr": 0.035670166752768635, | |
| "acc_norm": 0.1984126984126984, | |
| "acc_norm_stderr": 0.035670166752768635 | |
| }, | |
| "hendrycksTest-global_facts": { | |
| "acc": 0.34, | |
| "acc_stderr": 0.04760952285695236, | |
| "acc_norm": 0.34, | |
| "acc_norm_stderr": 0.04760952285695236 | |
| }, | |
| "hendrycksTest-high_school_biology": { | |
| "acc": 0.3225806451612903, | |
| "acc_stderr": 0.02659308451657229, | |
| "acc_norm": 0.3225806451612903, | |
| "acc_norm_stderr": 0.02659308451657229 | |
| }, | |
| "hendrycksTest-high_school_chemistry": { | |
| "acc": 0.2315270935960591, | |
| "acc_stderr": 0.02967833314144444, | |
| "acc_norm": 0.2315270935960591, | |
| "acc_norm_stderr": 0.02967833314144444 | |
| }, | |
| "hendrycksTest-high_school_computer_science": { | |
| "acc": 0.34, | |
| "acc_stderr": 0.047609522856952365, | |
| "acc_norm": 0.34, | |
| "acc_norm_stderr": 0.047609522856952365 | |
| }, | |
| "hendrycksTest-high_school_european_history": { | |
| "acc": 0.4, | |
| "acc_stderr": 0.03825460278380026, | |
| "acc_norm": 0.4, | |
| "acc_norm_stderr": 0.03825460278380026 | |
| }, | |
| "hendrycksTest-high_school_geography": { | |
| "acc": 0.3383838383838384, | |
| "acc_stderr": 0.03371124142626303, | |
| "acc_norm": 0.3383838383838384, | |
| "acc_norm_stderr": 0.03371124142626303 | |
| }, | |
| "hendrycksTest-high_school_government_and_politics": { | |
| "acc": 0.32642487046632124, | |
| "acc_stderr": 0.033840286211432945, | |
| "acc_norm": 0.32642487046632124, | |
| "acc_norm_stderr": 0.033840286211432945 | |
| }, | |
| "hendrycksTest-high_school_macroeconomics": { | |
| "acc": 0.2358974358974359, | |
| "acc_stderr": 0.021525965407408726, | |
| "acc_norm": 0.2358974358974359, | |
| "acc_norm_stderr": 0.021525965407408726 | |
| }, | |
| "hendrycksTest-high_school_mathematics": { | |
| "acc": 0.23703703703703705, | |
| "acc_stderr": 0.025928876132766104, | |
| "acc_norm": 0.23703703703703705, | |
| "acc_norm_stderr": 0.025928876132766104 | |
| }, | |
| "hendrycksTest-high_school_microeconomics": { | |
| "acc": 0.29411764705882354, | |
| "acc_stderr": 0.02959732973097808, | |
| "acc_norm": 0.29411764705882354, | |
| "acc_norm_stderr": 0.02959732973097808 | |
| }, | |
| "hendrycksTest-high_school_physics": { | |
| "acc": 0.2582781456953642, | |
| "acc_stderr": 0.035737053147634576, | |
| "acc_norm": 0.2582781456953642, | |
| "acc_norm_stderr": 0.035737053147634576 | |
| }, | |
| "hendrycksTest-high_school_psychology": { | |
| "acc": 0.3779816513761468, | |
| "acc_stderr": 0.020789187066728117, | |
| "acc_norm": 0.3779816513761468, | |
| "acc_norm_stderr": 0.020789187066728117 | |
| }, | |
| "hendrycksTest-high_school_statistics": { | |
| "acc": 0.1712962962962963, | |
| "acc_stderr": 0.025695341643824674, | |
| "acc_norm": 0.1712962962962963, | |
| "acc_norm_stderr": 0.025695341643824674 | |
| }, | |
| "hendrycksTest-high_school_us_history": { | |
| "acc": 0.3627450980392157, | |
| "acc_stderr": 0.03374499356319355, | |
| "acc_norm": 0.3627450980392157, | |
| "acc_norm_stderr": 0.03374499356319355 | |
| }, | |
| "hendrycksTest-high_school_world_history": { | |
| "acc": 0.47257383966244726, | |
| "acc_stderr": 0.03249822718301303, | |
| "acc_norm": 0.47257383966244726, | |
| "acc_norm_stderr": 0.03249822718301303 | |
| }, | |
| "hendrycksTest-human_aging": { | |
| "acc": 0.4080717488789238, | |
| "acc_stderr": 0.03298574607842821, | |
| "acc_norm": 0.4080717488789238, | |
| "acc_norm_stderr": 0.03298574607842821 | |
| }, | |
| "hendrycksTest-human_sexuality": { | |
| "acc": 0.3053435114503817, | |
| "acc_stderr": 0.040393149787245626, | |
| "acc_norm": 0.3053435114503817, | |
| "acc_norm_stderr": 0.040393149787245626 | |
| }, | |
| "hendrycksTest-international_law": { | |
| "acc": 0.5619834710743802, | |
| "acc_stderr": 0.045291468044357915, | |
| "acc_norm": 0.5619834710743802, | |
| "acc_norm_stderr": 0.045291468044357915 | |
| }, | |
| "hendrycksTest-jurisprudence": { | |
| "acc": 0.37037037037037035, | |
| "acc_stderr": 0.04668408033024932, | |
| "acc_norm": 0.37037037037037035, | |
| "acc_norm_stderr": 0.04668408033024932 | |
| }, | |
| "hendrycksTest-logical_fallacies": { | |
| "acc": 0.38650306748466257, | |
| "acc_stderr": 0.038258255488486076, | |
| "acc_norm": 0.38650306748466257, | |
| "acc_norm_stderr": 0.038258255488486076 | |
| }, | |
| "hendrycksTest-machine_learning": { | |
| "acc": 0.3482142857142857, | |
| "acc_stderr": 0.04521829902833585, | |
| "acc_norm": 0.3482142857142857, | |
| "acc_norm_stderr": 0.04521829902833585 | |
| }, | |
| "hendrycksTest-management": { | |
| "acc": 0.3300970873786408, | |
| "acc_stderr": 0.0465614711001235, | |
| "acc_norm": 0.3300970873786408, | |
| "acc_norm_stderr": 0.0465614711001235 | |
| }, | |
| "hendrycksTest-marketing": { | |
| "acc": 0.47435897435897434, | |
| "acc_stderr": 0.03271298896811159, | |
| "acc_norm": 0.47435897435897434, | |
| "acc_norm_stderr": 0.03271298896811159 | |
| }, | |
| "hendrycksTest-medical_genetics": { | |
| "acc": 0.35, | |
| "acc_stderr": 0.047937248544110196, | |
| "acc_norm": 0.35, | |
| "acc_norm_stderr": 0.047937248544110196 | |
| }, | |
| "hendrycksTest-miscellaneous": { | |
| "acc": 0.47126436781609193, | |
| "acc_stderr": 0.01785041079438017, | |
| "acc_norm": 0.47126436781609193, | |
| "acc_norm_stderr": 0.01785041079438017 | |
| }, | |
| "hendrycksTest-moral_disputes": { | |
| "acc": 0.315028901734104, | |
| "acc_stderr": 0.025009313790069727, | |
| "acc_norm": 0.315028901734104, | |
| "acc_norm_stderr": 0.025009313790069727 | |
| }, | |
| "hendrycksTest-moral_scenarios": { | |
| "acc": 0.23798882681564246, | |
| "acc_stderr": 0.014242630070574915, | |
| "acc_norm": 0.23798882681564246, | |
| "acc_norm_stderr": 0.014242630070574915 | |
| }, | |
| "hendrycksTest-nutrition": { | |
| "acc": 0.369281045751634, | |
| "acc_stderr": 0.027634176689602653, | |
| "acc_norm": 0.369281045751634, | |
| "acc_norm_stderr": 0.027634176689602653 | |
| }, | |
| "hendrycksTest-philosophy": { | |
| "acc": 0.3633440514469453, | |
| "acc_stderr": 0.027316847674192717, | |
| "acc_norm": 0.3633440514469453, | |
| "acc_norm_stderr": 0.027316847674192717 | |
| }, | |
| "hendrycksTest-prehistory": { | |
| "acc": 0.36419753086419754, | |
| "acc_stderr": 0.02677492989972233, | |
| "acc_norm": 0.36419753086419754, | |
| "acc_norm_stderr": 0.02677492989972233 | |
| }, | |
| "hendrycksTest-professional_accounting": { | |
| "acc": 0.30141843971631205, | |
| "acc_stderr": 0.02737412888263115, | |
| "acc_norm": 0.30141843971631205, | |
| "acc_norm_stderr": 0.02737412888263115 | |
| }, | |
| "hendrycksTest-professional_law": { | |
| "acc": 0.3044328552803129, | |
| "acc_stderr": 0.011752877592597575, | |
| "acc_norm": 0.3044328552803129, | |
| "acc_norm_stderr": 0.011752877592597575 | |
| }, | |
| "hendrycksTest-professional_medicine": { | |
| "acc": 0.2426470588235294, | |
| "acc_stderr": 0.026040662474201264, | |
| "acc_norm": 0.2426470588235294, | |
| "acc_norm_stderr": 0.026040662474201264 | |
| }, | |
| "hendrycksTest-professional_psychology": { | |
| "acc": 0.3104575163398693, | |
| "acc_stderr": 0.018718067052623227, | |
| "acc_norm": 0.3104575163398693, | |
| "acc_norm_stderr": 0.018718067052623227 | |
| }, | |
| "hendrycksTest-public_relations": { | |
| "acc": 0.38181818181818183, | |
| "acc_stderr": 0.04653429807913508, | |
| "acc_norm": 0.38181818181818183, | |
| "acc_norm_stderr": 0.04653429807913508 | |
| }, | |
| "hendrycksTest-security_studies": { | |
| "acc": 0.3020408163265306, | |
| "acc_stderr": 0.029393609319879818, | |
| "acc_norm": 0.3020408163265306, | |
| "acc_norm_stderr": 0.029393609319879818 | |
| }, | |
| "hendrycksTest-sociology": { | |
| "acc": 0.4079601990049751, | |
| "acc_stderr": 0.034751163651940926, | |
| "acc_norm": 0.4079601990049751, | |
| "acc_norm_stderr": 0.034751163651940926 | |
| }, | |
| "hendrycksTest-us_foreign_policy": { | |
| "acc": 0.41, | |
| "acc_stderr": 0.04943110704237102, | |
| "acc_norm": 0.41, | |
| "acc_norm_stderr": 0.04943110704237102 | |
| }, | |
| "hendrycksTest-virology": { | |
| "acc": 0.4036144578313253, | |
| "acc_stderr": 0.038194861407583984, | |
| "acc_norm": 0.4036144578313253, | |
| "acc_norm_stderr": 0.038194861407583984 | |
| }, | |
| "hendrycksTest-world_religions": { | |
| "acc": 0.40350877192982454, | |
| "acc_stderr": 0.03762738699917056, | |
| "acc_norm": 0.40350877192982454, | |
| "acc_norm_stderr": 0.03762738699917056 | |
| } | |
| }, | |
| "versions": { | |
| "hendrycksTest-abstract_algebra": 1, | |
| "hendrycksTest-anatomy": 1, | |
| "hendrycksTest-astronomy": 1, | |
| "hendrycksTest-business_ethics": 1, | |
| "hendrycksTest-clinical_knowledge": 1, | |
| "hendrycksTest-college_biology": 1, | |
| "hendrycksTest-college_chemistry": 1, | |
| "hendrycksTest-college_computer_science": 1, | |
| "hendrycksTest-college_mathematics": 1, | |
| "hendrycksTest-college_medicine": 1, | |
| "hendrycksTest-college_physics": 1, | |
| "hendrycksTest-computer_security": 1, | |
| "hendrycksTest-conceptual_physics": 1, | |
| "hendrycksTest-econometrics": 1, | |
| "hendrycksTest-electrical_engineering": 1, | |
| "hendrycksTest-elementary_mathematics": 1, | |
| "hendrycksTest-formal_logic": 1, | |
| "hendrycksTest-global_facts": 1, | |
| "hendrycksTest-high_school_biology": 1, | |
| "hendrycksTest-high_school_chemistry": 1, | |
| "hendrycksTest-high_school_computer_science": 1, | |
| "hendrycksTest-high_school_european_history": 1, | |
| "hendrycksTest-high_school_geography": 1, | |
| "hendrycksTest-high_school_government_and_politics": 1, | |
| "hendrycksTest-high_school_macroeconomics": 1, | |
| "hendrycksTest-high_school_mathematics": 1, | |
| "hendrycksTest-high_school_microeconomics": 1, | |
| "hendrycksTest-high_school_physics": 1, | |
| "hendrycksTest-high_school_psychology": 1, | |
| "hendrycksTest-high_school_statistics": 1, | |
| "hendrycksTest-high_school_us_history": 1, | |
| "hendrycksTest-high_school_world_history": 1, | |
| "hendrycksTest-human_aging": 1, | |
| "hendrycksTest-human_sexuality": 1, | |
| "hendrycksTest-international_law": 1, | |
| "hendrycksTest-jurisprudence": 1, | |
| "hendrycksTest-logical_fallacies": 1, | |
| "hendrycksTest-machine_learning": 1, | |
| "hendrycksTest-management": 1, | |
| "hendrycksTest-marketing": 1, | |
| "hendrycksTest-medical_genetics": 1, | |
| "hendrycksTest-miscellaneous": 1, | |
| "hendrycksTest-moral_disputes": 1, | |
| "hendrycksTest-moral_scenarios": 1, | |
| "hendrycksTest-nutrition": 1, | |
| "hendrycksTest-philosophy": 1, | |
| "hendrycksTest-prehistory": 1, | |
| "hendrycksTest-professional_accounting": 1, | |
| "hendrycksTest-professional_law": 1, | |
| "hendrycksTest-professional_medicine": 1, | |
| "hendrycksTest-professional_psychology": 1, | |
| "hendrycksTest-public_relations": 1, | |
| "hendrycksTest-security_studies": 1, | |
| "hendrycksTest-sociology": 1, | |
| "hendrycksTest-us_foreign_policy": 1, | |
| "hendrycksTest-virology": 1, | |
| "hendrycksTest-world_religions": 1 | |
| }, | |
| "config": { | |
| "model": "sparseml", | |
| "model_args": "pretrained=/network/alexandre/research/cerebras/llama2_7B_sparse70_retrained/ultrachat200k/llama2_7B_sparse70_LR3e-4_GC2_E2/training,dtype=bfloat16", | |
| "num_fewshot": 5, | |
| "batch_size": "4", | |
| "batch_sizes": [], | |
| "device": "cuda:7", | |
| "no_cache": true, | |
| "limit": null, | |
| "bootstrap_iters": 100000, | |
| "description_dict": {} | |
| } | |
| } |