| { | |
| "results": { | |
| "assin2_rte": { | |
| "f1_macro,all": 0.6778064209006744, | |
| "f1_macro_stderr,all": 0.006813205507001969, | |
| "acc,all": 0.6907679738562091, | |
| "acc_stderr,all": 0.006616054688908941, | |
| "alias": "assin2_rte" | |
| }, | |
| "assin2_sts": { | |
| "pearson,all": 0.08876834100904987, | |
| "pearson_stderr,all": 0.013601135542002981, | |
| "mse,all": 1.4245261437908499, | |
| "mse_stderr,all": "N/A", | |
| "alias": "assin2_sts" | |
| }, | |
| "bluex": { | |
| "acc,all": 0.26008344923504867, | |
| "acc_stderr,all": 0.009452533294580755, | |
| "acc,exam_id__USP_2023": 0.3181818181818182, | |
| "acc_stderr,exam_id__USP_2023": 0.04052437035850368, | |
| "acc,exam_id__UNICAMP_2021_2": 0.29411764705882354, | |
| "acc_stderr,exam_id__UNICAMP_2021_2": 0.03678406842196161, | |
| "acc,exam_id__USP_2021": 0.19230769230769232, | |
| "acc_stderr,exam_id__USP_2021": 0.03147378966283576, | |
| "acc,exam_id__UNICAMP_2019": 0.26, | |
| "acc_stderr,exam_id__UNICAMP_2019": 0.03585259404178158, | |
| "acc,exam_id__UNICAMP_2020": 0.32727272727272727, | |
| "acc_stderr,exam_id__UNICAMP_2020": 0.03655630277825746, | |
| "acc,exam_id__USP_2018": 0.18518518518518517, | |
| "acc_stderr,exam_id__USP_2018": 0.030596318881900852, | |
| "acc,exam_id__USP_2024": 0.17073170731707318, | |
| "acc_stderr,exam_id__USP_2024": 0.03378335774802068, | |
| "acc,exam_id__USP_2019": 0.225, | |
| "acc_stderr,exam_id__USP_2019": 0.03818292580701932, | |
| "acc,exam_id__UNICAMP_2021_1": 0.32608695652173914, | |
| "acc_stderr,exam_id__UNICAMP_2021_1": 0.03987627273416352, | |
| "acc,exam_id__UNICAMP_2022": 0.28205128205128205, | |
| "acc_stderr,exam_id__UNICAMP_2022": 0.04167152779427693, | |
| "acc,exam_id__UNICAMP_2023": 0.23255813953488372, | |
| "acc_stderr,exam_id__UNICAMP_2023": 0.03716515848328401, | |
| "acc,exam_id__UNICAMP_2024": 0.37777777777777777, | |
| "acc_stderr,exam_id__UNICAMP_2024": 0.04167427031074604, | |
| "acc,exam_id__USP_2020": 0.25, | |
| "acc_stderr,exam_id__USP_2020": 0.03348879615541624, | |
| "acc,exam_id__USP_2022": 0.20408163265306123, | |
| "acc_stderr,exam_id__USP_2022": 0.03327189738781395, | |
| "acc,exam_id__UNICAMP_2018": 0.25925925925925924, | |
| "acc_stderr,exam_id__UNICAMP_2018": 0.03440732040215374, | |
| "alias": "bluex" | |
| }, | |
| "enem_challenge": { | |
| "alias": "enem", | |
| "acc,all": 0.2106368089573128, | |
| "acc_stderr,all": 0.006221926886428458, | |
| "acc,exam_id__2023": 0.1925925925925926, | |
| "acc_stderr,exam_id__2023": 0.019642560525314663, | |
| "acc,exam_id__2016": 0.2231404958677686, | |
| "acc_stderr,exam_id__2016": 0.021884506550252786, | |
| "acc,exam_id__2011": 0.20512820512820512, | |
| "acc_stderr,exam_id__2011": 0.021535615178630927, | |
| "acc,exam_id__2014": 0.2018348623853211, | |
| "acc_stderr,exam_id__2014": 0.02227514909979303, | |
| "acc,exam_id__2016_2": 0.18699186991869918, | |
| "acc_stderr,exam_id__2016_2": 0.020236833190250755, | |
| "acc,exam_id__2013": 0.23148148148148148, | |
| "acc_stderr,exam_id__2013": 0.023403559519633636, | |
| "acc,exam_id__2022": 0.19548872180451127, | |
| "acc_stderr,exam_id__2022": 0.0197906677810766, | |
| "acc,exam_id__2017": 0.25862068965517243, | |
| "acc_stderr,exam_id__2017": 0.023462571285656597, | |
| "acc,exam_id__2009": 0.19130434782608696, | |
| "acc_stderr,exam_id__2009": 0.021192171541977615, | |
| "acc,exam_id__2012": 0.20689655172413793, | |
| "acc_stderr,exam_id__2012": 0.02164777225944937, | |
| "acc,exam_id__2010": 0.18803418803418803, | |
| "acc_stderr,exam_id__2010": 0.02084241608479345, | |
| "acc,exam_id__2015": 0.25210084033613445, | |
| "acc_stderr,exam_id__2015": 0.02302550703391277 | |
| }, | |
| "faquad_nli": { | |
| "f1_macro,all": 0.4396551724137931, | |
| "f1_macro_stderr,all": 0.0035796984729087084, | |
| "acc,all": 0.7846153846153846, | |
| "acc_stderr,all": 0.011396120309131327, | |
| "alias": "faquad_nli" | |
| }, | |
| "hatebr_offensive": { | |
| "alias": "hatebr_offensive_binary", | |
| "f1_macro,all": 0.3128178313648294, | |
| "f1_macro_stderr,all": 0.004586812204783609, | |
| "acc,all": 0.45285714285714285, | |
| "acc_stderr,all": 0.009431973605901981 | |
| }, | |
| "oab_exams": { | |
| "acc,all": 0.26469248291571756, | |
| "acc_stderr,all": 0.005428752815268037, | |
| "acc,exam_id__2013-12": 0.275, | |
| "acc_stderr,exam_id__2013-12": 0.02884127035012301, | |
| "acc,exam_id__2015-18": 0.2, | |
| "acc_stderr,exam_id__2015-18": 0.025728334229438835, | |
| "acc,exam_id__2017-24": 0.275, | |
| "acc_stderr,exam_id__2017-24": 0.028829140552922117, | |
| "acc,exam_id__2012-06a": 0.25, | |
| "acc_stderr,exam_id__2012-06a": 0.027921672471627818, | |
| "acc,exam_id__2016-19": 0.2564102564102564, | |
| "acc_stderr,exam_id__2016-19": 0.0285751477773131, | |
| "acc,exam_id__2011-04": 0.2625, | |
| "acc_stderr,exam_id__2011-04": 0.028487240212834847, | |
| "acc,exam_id__2016-20a": 0.225, | |
| "acc_stderr,exam_id__2016-20a": 0.026968247099866234, | |
| "acc,exam_id__2011-03": 0.25252525252525254, | |
| "acc_stderr,exam_id__2011-03": 0.02517984462055549, | |
| "acc,exam_id__2012-08": 0.325, | |
| "acc_stderr,exam_id__2012-08": 0.03021485499773354, | |
| "acc,exam_id__2012-09": 0.2987012987012987, | |
| "acc_stderr,exam_id__2012-09": 0.030147139671855144, | |
| "acc,exam_id__2016-20": 0.225, | |
| "acc_stderr,exam_id__2016-20": 0.026862967483280497, | |
| "acc,exam_id__2016-21": 0.2875, | |
| "acc_stderr,exam_id__2016-21": 0.029172210122811244, | |
| "acc,exam_id__2017-22": 0.2375, | |
| "acc_stderr,exam_id__2017-22": 0.027472473430105962, | |
| "acc,exam_id__2018-25": 0.275, | |
| "acc_stderr,exam_id__2018-25": 0.028925570890046506, | |
| "acc,exam_id__2013-11": 0.325, | |
| "acc_stderr,exam_id__2013-11": 0.030308086980619833, | |
| "acc,exam_id__2010-02": 0.25, | |
| "acc_stderr,exam_id__2010-02": 0.025044458442253486, | |
| "acc,exam_id__2012-07": 0.3125, | |
| "acc_stderr,exam_id__2012-07": 0.029891600574488583, | |
| "acc,exam_id__2015-17": 0.24358974358974358, | |
| "acc_stderr,exam_id__2015-17": 0.02805128221433091, | |
| "acc,exam_id__2010-01": 0.23529411764705882, | |
| "acc_stderr,exam_id__2010-01": 0.026627603995255614, | |
| "acc,exam_id__2014-14": 0.3125, | |
| "acc_stderr,exam_id__2014-14": 0.02988020366791311, | |
| "acc,exam_id__2015-16": 0.225, | |
| "acc_stderr,exam_id__2015-16": 0.02698715637840771, | |
| "acc,exam_id__2012-06": 0.2375, | |
| "acc_stderr,exam_id__2012-06": 0.027469022000536025, | |
| "acc,exam_id__2011-05": 0.25, | |
| "acc_stderr,exam_id__2011-05": 0.027880232512179248, | |
| "acc,exam_id__2014-15": 0.3333333333333333, | |
| "acc_stderr,exam_id__2014-15": 0.03076413057616716, | |
| "acc,exam_id__2013-10": 0.3125, | |
| "acc_stderr,exam_id__2013-10": 0.029820761573406854, | |
| "acc,exam_id__2014-13": 0.25, | |
| "acc_stderr,exam_id__2014-13": 0.02798578403861933, | |
| "acc,exam_id__2017-23": 0.225, | |
| "acc_stderr,exam_id__2017-23": 0.026939295289475033, | |
| "alias": "oab_exams" | |
| }, | |
| "portuguese_hate_speech": { | |
| "alias": "portuguese_hate_speech_binary", | |
| "f1_macro,all": 0.412292817679558, | |
| "f1_macro_stderr,all": 0.0038204811103947646, | |
| "acc,all": 0.7015276145710928, | |
| "acc_stderr,all": 0.011059935171649318 | |
| }, | |
| "tweetsentbr": { | |
| "f1_macro,all": 0.22033834583581732, | |
| "f1_macro_stderr,all": 0.003670253679085497, | |
| "acc,all": 0.45671641791044776, | |
| "acc_stderr,all": 0.007871122638710882, | |
| "alias": "tweetsentbr" | |
| } | |
| }, | |
| "configs": { | |
| "assin2_rte": { | |
| "task": "assin2_rte", | |
| "group": [ | |
| "pt_benchmark", | |
| "assin2" | |
| ], | |
| "dataset_path": "assin2", | |
| "test_split": "test", | |
| "fewshot_split": "train", | |
| "doc_to_text": "Premissa: {{premise}}\nHipótese: {{hypothesis}}\nPergunta: A hipótese pode ser inferida pela premissa? Sim ou Não?\nResposta:", | |
| "doc_to_target": "{{['Não', 'Sim'][entailment_judgment]}}", | |
| "description": "Abaixo estão pares de premissa e hipótese. Para cada par, indique se a hipótese pode ser inferida a partir da premissa, responda apenas com \"Sim\" ou \"Não\".\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "id_sampler", | |
| "sampler_config": { | |
| "id_list": [ | |
| 1, | |
| 3251, | |
| 2, | |
| 3252, | |
| 3, | |
| 4, | |
| 5, | |
| 6, | |
| 3253, | |
| 7, | |
| 3254, | |
| 3255, | |
| 3256, | |
| 8, | |
| 9, | |
| 10, | |
| 3257, | |
| 11, | |
| 3258, | |
| 12, | |
| 13, | |
| 14, | |
| 15, | |
| 3259, | |
| 3260, | |
| 3261, | |
| 3262, | |
| 3263, | |
| 16, | |
| 17, | |
| 3264, | |
| 18, | |
| 3265, | |
| 3266, | |
| 3267, | |
| 19, | |
| 20, | |
| 3268, | |
| 3269, | |
| 21, | |
| 3270, | |
| 3271, | |
| 22, | |
| 3272, | |
| 3273, | |
| 23, | |
| 3274, | |
| 24, | |
| 25, | |
| 3275 | |
| ], | |
| "id_column": "sentence_pair_id" | |
| } | |
| }, | |
| "num_fewshot": 15, | |
| "metric_list": [ | |
| { | |
| "metric": "f1_macro", | |
| "aggregation": "f1_macro", | |
| "higher_is_better": true | |
| }, | |
| { | |
| "metric": "acc", | |
| "aggregation": "acc", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "generate_until", | |
| "generation_kwargs": { | |
| "max_gen_toks": 32, | |
| "do_sample": false, | |
| "temperature": 0.0, | |
| "top_k": null, | |
| "top_p": null, | |
| "until": [ | |
| "\n\n" | |
| ] | |
| }, | |
| "repeats": 1, | |
| "filter_list": [ | |
| { | |
| "name": "all", | |
| "filter": [ | |
| { | |
| "function": "find_similar_label", | |
| "labels": [ | |
| "Sim", | |
| "Não" | |
| ] | |
| }, | |
| { | |
| "function": "take_first" | |
| } | |
| ] | |
| } | |
| ], | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.1 | |
| } | |
| }, | |
| "assin2_sts": { | |
| "task": "assin2_sts", | |
| "group": [ | |
| "pt_benchmark", | |
| "assin2" | |
| ], | |
| "dataset_path": "assin2", | |
| "test_split": "test", | |
| "fewshot_split": "train", | |
| "doc_to_text": "Frase 1: {{premise}}\nFrase 2: {{hypothesis}}\nPergunta: Quão similares são as duas frases? Dê uma pontuação entre 1,0 a 5,0.\nResposta:", | |
| "doc_to_target": "<function assin2_float_to_pt_str at 0x1493ec0de5c0>", | |
| "description": "Abaixo estão pares de frases que você deve avaliar o grau de similaridade. Dê uma pontuação entre 1,0 e 5,0, sendo 1,0 pouco similar e 5,0 muito similar.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "id_sampler", | |
| "sampler_config": { | |
| "id_list": [ | |
| 1, | |
| 3251, | |
| 2, | |
| 3252, | |
| 3, | |
| 4, | |
| 5, | |
| 6, | |
| 3253, | |
| 7, | |
| 3254, | |
| 3255, | |
| 3256, | |
| 8, | |
| 9, | |
| 10, | |
| 3257, | |
| 11, | |
| 3258, | |
| 12, | |
| 13, | |
| 14, | |
| 15, | |
| 3259, | |
| 3260, | |
| 3261, | |
| 3262, | |
| 3263, | |
| 16, | |
| 17, | |
| 3264, | |
| 18, | |
| 3265, | |
| 3266, | |
| 3267, | |
| 19, | |
| 20, | |
| 3268, | |
| 3269, | |
| 21, | |
| 3270, | |
| 3271, | |
| 22, | |
| 3272, | |
| 3273, | |
| 23, | |
| 3274, | |
| 24, | |
| 25, | |
| 3275 | |
| ], | |
| "id_column": "sentence_pair_id" | |
| } | |
| }, | |
| "num_fewshot": 10, | |
| "metric_list": [ | |
| { | |
| "metric": "pearson", | |
| "aggregation": "pearsonr", | |
| "higher_is_better": true | |
| }, | |
| { | |
| "metric": "mse", | |
| "aggregation": "mean_squared_error", | |
| "higher_is_better": false | |
| } | |
| ], | |
| "output_type": "generate_until", | |
| "generation_kwargs": { | |
| "max_gen_toks": 32, | |
| "do_sample": false, | |
| "temperature": 0.0, | |
| "top_k": null, | |
| "top_p": null, | |
| "until": [ | |
| "\n\n" | |
| ] | |
| }, | |
| "repeats": 1, | |
| "filter_list": [ | |
| { | |
| "name": "all", | |
| "filter": [ | |
| { | |
| "function": "number_filter", | |
| "type": "float", | |
| "range_min": 1.0, | |
| "range_max": 5.0, | |
| "on_outside_range": "clip", | |
| "fallback": 5.0 | |
| }, | |
| { | |
| "function": "take_first" | |
| } | |
| ] | |
| } | |
| ], | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.1 | |
| } | |
| }, | |
| "bluex": { | |
| "task": "bluex", | |
| "group": [ | |
| "pt_benchmark", | |
| "vestibular" | |
| ], | |
| "dataset_path": "eduagarcia-temp/BLUEX_without_images", | |
| "test_split": "train", | |
| "fewshot_split": "train", | |
| "doc_to_text": "<function enem_doc_to_text at 0x1493ec0ddb20>", | |
| "doc_to_target": "{{answerKey}}", | |
| "description": "As perguntas a seguir são questões de múltipla escolha de provas de vestibular de universidades brasileiras, selecione a única alternativa correta e responda apenas com as letras \"A\", \"B\", \"C\", \"D\" ou \"E\".\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "id_sampler", | |
| "sampler_config": { | |
| "id_list": [ | |
| "USP_2018_3", | |
| "UNICAMP_2018_2", | |
| "USP_2018_35", | |
| "UNICAMP_2018_16", | |
| "USP_2018_89" | |
| ], | |
| "id_column": "id", | |
| "exclude_from_task": true | |
| } | |
| }, | |
| "num_fewshot": 3, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "acc", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "generate_until", | |
| "generation_kwargs": { | |
| "max_gen_toks": 32, | |
| "do_sample": false, | |
| "temperature": 0.0, | |
| "top_k": null, | |
| "top_p": null, | |
| "until": [ | |
| "\n\n" | |
| ] | |
| }, | |
| "repeats": 1, | |
| "filter_list": [ | |
| { | |
| "name": "all", | |
| "filter": [ | |
| { | |
| "function": "normalize_spaces" | |
| }, | |
| { | |
| "function": "remove_accents" | |
| }, | |
| { | |
| "function": "find_choices", | |
| "choices": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D", | |
| "E" | |
| ], | |
| "regex_patterns": [ | |
| "(?:[Ll]etra|[Aa]lternativa|[Rr]esposta|[Rr]esposta [Cc]orreta|[Rr]esposta [Cc]orreta e|[Oo]pcao):? ([ABCDE])\\b", | |
| "\\b([ABCDE])\\.", | |
| "\\b([ABCDE]) ?[.):-]", | |
| "\\b([ABCDE])$", | |
| "\\b([ABCDE])\\b" | |
| ] | |
| }, | |
| { | |
| "function": "take_first" | |
| } | |
| ], | |
| "group_by": { | |
| "column": "exam_id" | |
| } | |
| } | |
| ], | |
| "should_decontaminate": true, | |
| "doc_to_decontamination_query": "<function enem_doc_to_text at 0x1493ec0dde40>", | |
| "metadata": { | |
| "version": 1.1 | |
| } | |
| }, | |
| "enem_challenge": { | |
| "task": "enem_challenge", | |
| "task_alias": "enem", | |
| "group": [ | |
| "pt_benchmark", | |
| "vestibular" | |
| ], | |
| "dataset_path": "eduagarcia/enem_challenge", | |
| "test_split": "train", | |
| "fewshot_split": "train", | |
| "doc_to_text": "<function enem_doc_to_text at 0x1493ec0de020>", | |
| "doc_to_target": "{{answerKey}}", | |
| "description": "As perguntas a seguir são questões de múltipla escolha do Exame Nacional do Ensino Médio (ENEM), selecione a única alternativa correta e responda apenas com as letras \"A\", \"B\", \"C\", \"D\" ou \"E\".\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "id_sampler", | |
| "sampler_config": { | |
| "id_list": [ | |
| "2022_21", | |
| "2022_88", | |
| "2022_143" | |
| ], | |
| "id_column": "id", | |
| "exclude_from_task": true | |
| } | |
| }, | |
| "num_fewshot": 3, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "acc", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "generate_until", | |
| "generation_kwargs": { | |
| "max_gen_toks": 32, | |
| "do_sample": false, | |
| "temperature": 0.0, | |
| "top_k": null, | |
| "top_p": null, | |
| "until": [ | |
| "\n\n" | |
| ] | |
| }, | |
| "repeats": 1, | |
| "filter_list": [ | |
| { | |
| "name": "all", | |
| "filter": [ | |
| { | |
| "function": "normalize_spaces" | |
| }, | |
| { | |
| "function": "remove_accents" | |
| }, | |
| { | |
| "function": "find_choices", | |
| "choices": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D", | |
| "E" | |
| ], | |
| "regex_patterns": [ | |
| "(?:[Ll]etra|[Aa]lternativa|[Rr]esposta|[Rr]esposta [Cc]orreta|[Rr]esposta [Cc]orreta e|[Oo]pcao):? ([ABCDE])\\b", | |
| "\\b([ABCDE])\\.", | |
| "\\b([ABCDE]) ?[.):-]", | |
| "\\b([ABCDE])$", | |
| "\\b([ABCDE])\\b" | |
| ] | |
| }, | |
| { | |
| "function": "take_first" | |
| } | |
| ], | |
| "group_by": { | |
| "column": "exam_id" | |
| } | |
| } | |
| ], | |
| "should_decontaminate": true, | |
| "doc_to_decontamination_query": "<function enem_doc_to_text at 0x1493ec0de200>", | |
| "metadata": { | |
| "version": 1.1 | |
| } | |
| }, | |
| "faquad_nli": { | |
| "task": "faquad_nli", | |
| "group": [ | |
| "pt_benchmark" | |
| ], | |
| "dataset_path": "ruanchaves/faquad-nli", | |
| "test_split": "test", | |
| "fewshot_split": "train", | |
| "doc_to_text": "Pergunta: {{question}}\nResposta: {{answer}}\nA resposta dada satisfaz à pergunta? Sim ou Não?", | |
| "doc_to_target": "{{['Não', 'Sim'][label]}}", | |
| "description": "Abaixo estão pares de pergunta e resposta. Para cada par, você deve julgar se a resposta responde à pergunta de maneira satisfatória e aparenta estar correta. Escreva apenas \"Sim\" ou \"Não\".\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n", | |
| "sampler_config": { | |
| "fewshot_indices": [ | |
| 1893, | |
| 949, | |
| 663, | |
| 105, | |
| 1169, | |
| 2910, | |
| 2227, | |
| 2813, | |
| 974, | |
| 558, | |
| 1503, | |
| 1958, | |
| 2918, | |
| 601, | |
| 1560, | |
| 984, | |
| 2388, | |
| 995, | |
| 2233, | |
| 1982, | |
| 165, | |
| 2788, | |
| 1312, | |
| 2285, | |
| 522, | |
| 1113, | |
| 1670, | |
| 323, | |
| 236, | |
| 1263, | |
| 1562, | |
| 2519, | |
| 1049, | |
| 432, | |
| 1167, | |
| 1394, | |
| 2022, | |
| 2551, | |
| 2194, | |
| 2187, | |
| 2282, | |
| 2816, | |
| 108, | |
| 301, | |
| 1185, | |
| 1315, | |
| 1420, | |
| 2436, | |
| 2322, | |
| 766 | |
| ] | |
| } | |
| }, | |
| "num_fewshot": 15, | |
| "metric_list": [ | |
| { | |
| "metric": "f1_macro", | |
| "aggregation": "f1_macro", | |
| "higher_is_better": true | |
| }, | |
| { | |
| "metric": "acc", | |
| "aggregation": "acc", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "generate_until", | |
| "generation_kwargs": { | |
| "max_gen_toks": 32, | |
| "do_sample": false, | |
| "temperature": 0.0, | |
| "top_k": null, | |
| "top_p": null, | |
| "until": [ | |
| "\n\n" | |
| ] | |
| }, | |
| "repeats": 1, | |
| "filter_list": [ | |
| { | |
| "name": "all", | |
| "filter": [ | |
| { | |
| "function": "find_similar_label", | |
| "labels": [ | |
| "Sim", | |
| "Não" | |
| ] | |
| }, | |
| { | |
| "function": "take_first" | |
| } | |
| ] | |
| } | |
| ], | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.1 | |
| } | |
| }, | |
| "hatebr_offensive": { | |
| "task": "hatebr_offensive", | |
| "task_alias": "hatebr_offensive_binary", | |
| "group": [ | |
| "pt_benchmark" | |
| ], | |
| "dataset_path": "eduagarcia/portuguese_benchmark", | |
| "dataset_name": "HateBR_offensive_binary", | |
| "test_split": "test", | |
| "fewshot_split": "train", | |
| "doc_to_text": "Texto: {{sentence}}\nPergunta: O texto é ofensivo?\nResposta:", | |
| "doc_to_target": "{{'Sim' if label == 1 else 'Não'}}", | |
| "description": "Abaixo contém o texto de comentários de usuários do Instagram em português, sua tarefa é classificar se o texto é ofensivo ou não. Responda apenas com \"Sim\" ou \"Não\".\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "id_sampler", | |
| "sampler_config": { | |
| "id_list": [ | |
| 48, | |
| 44, | |
| 36, | |
| 20, | |
| 3511, | |
| 88, | |
| 3555, | |
| 16, | |
| 56, | |
| 3535, | |
| 60, | |
| 40, | |
| 3527, | |
| 4, | |
| 76, | |
| 3579, | |
| 3523, | |
| 3551, | |
| 68, | |
| 3503, | |
| 84, | |
| 3539, | |
| 64, | |
| 3599, | |
| 80, | |
| 3563, | |
| 3559, | |
| 3543, | |
| 3547, | |
| 3587, | |
| 3595, | |
| 3575, | |
| 3567, | |
| 3591, | |
| 24, | |
| 96, | |
| 92, | |
| 3507, | |
| 52, | |
| 72, | |
| 8, | |
| 3571, | |
| 3515, | |
| 3519, | |
| 3531, | |
| 28, | |
| 32, | |
| 0, | |
| 12, | |
| 3583 | |
| ], | |
| "id_column": "idx" | |
| } | |
| }, | |
| "num_fewshot": 25, | |
| "metric_list": [ | |
| { | |
| "metric": "f1_macro", | |
| "aggregation": "f1_macro", | |
| "higher_is_better": true | |
| }, | |
| { | |
| "metric": "acc", | |
| "aggregation": "acc", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "generate_until", | |
| "generation_kwargs": { | |
| "max_gen_toks": 32, | |
| "do_sample": false, | |
| "temperature": 0.0, | |
| "top_k": null, | |
| "top_p": null, | |
| "until": [ | |
| "\n\n" | |
| ] | |
| }, | |
| "repeats": 1, | |
| "filter_list": [ | |
| { | |
| "name": "all", | |
| "filter": [ | |
| { | |
| "function": "find_similar_label", | |
| "labels": [ | |
| "Sim", | |
| "Não" | |
| ] | |
| }, | |
| { | |
| "function": "take_first" | |
| } | |
| ] | |
| } | |
| ], | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "oab_exams": { | |
| "task": "oab_exams", | |
| "group": [ | |
| "legal_benchmark", | |
| "pt_benchmark" | |
| ], | |
| "dataset_path": "eduagarcia/oab_exams", | |
| "test_split": "train", | |
| "fewshot_split": "train", | |
| "doc_to_text": "<function doc_to_text at 0x1493ec0ded40>", | |
| "doc_to_target": "{{answerKey}}", | |
| "description": "As perguntas a seguir são questões de múltipla escolha do Exame de Ordem da Ordem dos Advogados do Brasil (OAB), selecione a única alternativa correta e responda apenas com as letras \"A\", \"B\", \"C\" ou \"D\".\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "id_sampler", | |
| "sampler_config": { | |
| "id_list": [ | |
| "2010-01_1", | |
| "2010-01_11", | |
| "2010-01_13", | |
| "2010-01_23", | |
| "2010-01_26", | |
| "2010-01_28", | |
| "2010-01_38", | |
| "2010-01_48", | |
| "2010-01_58", | |
| "2010-01_68", | |
| "2010-01_76", | |
| "2010-01_83", | |
| "2010-01_85", | |
| "2010-01_91", | |
| "2010-01_99" | |
| ], | |
| "id_column": "id", | |
| "exclude_from_task": true | |
| } | |
| }, | |
| "num_fewshot": 3, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "acc", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "generate_until", | |
| "generation_kwargs": { | |
| "max_gen_toks": 32, | |
| "do_sample": false, | |
| "temperature": 0.0, | |
| "top_k": null, | |
| "top_p": null, | |
| "until": [ | |
| "\n\n" | |
| ] | |
| }, | |
| "repeats": 1, | |
| "filter_list": [ | |
| { | |
| "name": "all", | |
| "filter": [ | |
| { | |
| "function": "normalize_spaces" | |
| }, | |
| { | |
| "function": "remove_accents" | |
| }, | |
| { | |
| "function": "find_choices", | |
| "choices": [ | |
| "A", | |
| "B", | |
| "C", | |
| "D" | |
| ], | |
| "regex_patterns": [ | |
| "(?:[Ll]etra|[Aa]lternativa|[Rr]esposta|[Rr]esposta [Cc]orreta|[Rr]esposta [Cc]orreta e|[Oo]pcao):? ([ABCD])\\b", | |
| "\\b([ABCD])\\.", | |
| "\\b([ABCD]) ?[.):-]", | |
| "\\b([ABCD])$", | |
| "\\b([ABCD])\\b" | |
| ] | |
| }, | |
| { | |
| "function": "take_first" | |
| } | |
| ], | |
| "group_by": { | |
| "column": "exam_id" | |
| } | |
| } | |
| ], | |
| "should_decontaminate": true, | |
| "doc_to_decontamination_query": "<function doc_to_text at 0x1493ec0defc0>", | |
| "metadata": { | |
| "version": 1.5 | |
| } | |
| }, | |
| "portuguese_hate_speech": { | |
| "task": "portuguese_hate_speech", | |
| "task_alias": "portuguese_hate_speech_binary", | |
| "group": [ | |
| "pt_benchmark" | |
| ], | |
| "dataset_path": "eduagarcia/portuguese_benchmark", | |
| "dataset_name": "Portuguese_Hate_Speech_binary", | |
| "test_split": "test", | |
| "fewshot_split": "train", | |
| "doc_to_text": "Texto: {{sentence}}\nPergunta: O texto contém discurso de ódio?\nResposta:", | |
| "doc_to_target": "{{'Sim' if label == 1 else 'Não'}}", | |
| "description": "Abaixo contém o texto de tweets de usuários do Twitter em português, sua tarefa é classificar se o texto contém discurso de ódio ou não. Responda apenas com \"Sim\" ou \"Não\".\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "id_sampler", | |
| "sampler_config": { | |
| "id_list": [ | |
| 52, | |
| 50, | |
| 39, | |
| 28, | |
| 3, | |
| 105, | |
| 22, | |
| 25, | |
| 60, | |
| 11, | |
| 66, | |
| 41, | |
| 9, | |
| 4, | |
| 91, | |
| 42, | |
| 7, | |
| 20, | |
| 76, | |
| 1, | |
| 104, | |
| 13, | |
| 67, | |
| 54, | |
| 97, | |
| 27, | |
| 24, | |
| 14, | |
| 16, | |
| 48, | |
| 53, | |
| 40, | |
| 34, | |
| 49, | |
| 32, | |
| 119, | |
| 114, | |
| 2, | |
| 58, | |
| 83, | |
| 18, | |
| 36, | |
| 5, | |
| 6, | |
| 10, | |
| 35, | |
| 38, | |
| 0, | |
| 21, | |
| 46 | |
| ], | |
| "id_column": "idx" | |
| } | |
| }, | |
| "num_fewshot": 25, | |
| "metric_list": [ | |
| { | |
| "metric": "f1_macro", | |
| "aggregation": "f1_macro", | |
| "higher_is_better": true | |
| }, | |
| { | |
| "metric": "acc", | |
| "aggregation": "acc", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "generate_until", | |
| "generation_kwargs": { | |
| "max_gen_toks": 32, | |
| "do_sample": false, | |
| "temperature": 0.0, | |
| "top_k": null, | |
| "top_p": null, | |
| "until": [ | |
| "\n\n" | |
| ] | |
| }, | |
| "repeats": 1, | |
| "filter_list": [ | |
| { | |
| "name": "all", | |
| "filter": [ | |
| { | |
| "function": "find_similar_label", | |
| "labels": [ | |
| "Sim", | |
| "Não" | |
| ] | |
| }, | |
| { | |
| "function": "take_first" | |
| } | |
| ] | |
| } | |
| ], | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| }, | |
| "tweetsentbr": { | |
| "task": "tweetsentbr", | |
| "group": [ | |
| "pt_benchmark" | |
| ], | |
| "dataset_path": "eduagarcia/tweetsentbr_fewshot", | |
| "test_split": "test", | |
| "fewshot_split": "train", | |
| "doc_to_text": "Texto: {{sentence}}\nPergunta: O sentimento do texto é Positivo, Neutro ou Negativo?\nResposta:", | |
| "doc_to_target": "{{'Positivo' if label == 'Positive' else ('Negativo' if label == 'Negative' else 'Neutro')}}", | |
| "description": "Abaixo contém o texto de tweets de usuários do Twitter em português, sua tarefa é classificar se o sentimento do texto é Positivo, Neutro ou Negativo. Responda apenas com uma das opções.\n\n", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "fewshot_config": { | |
| "sampler": "first_n" | |
| }, | |
| "num_fewshot": 25, | |
| "metric_list": [ | |
| { | |
| "metric": "f1_macro", | |
| "aggregation": "f1_macro", | |
| "higher_is_better": true | |
| }, | |
| { | |
| "metric": "acc", | |
| "aggregation": "acc", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "generate_until", | |
| "generation_kwargs": { | |
| "max_gen_toks": 32, | |
| "do_sample": false, | |
| "temperature": 0.0, | |
| "top_k": null, | |
| "top_p": null, | |
| "until": [ | |
| "\n\n" | |
| ] | |
| }, | |
| "repeats": 1, | |
| "filter_list": [ | |
| { | |
| "name": "all", | |
| "filter": [ | |
| { | |
| "function": "find_similar_label", | |
| "labels": [ | |
| "Positivo", | |
| "Neutro", | |
| "Negativo" | |
| ] | |
| }, | |
| { | |
| "function": "take_first" | |
| } | |
| ] | |
| } | |
| ], | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| } | |
| }, | |
| "versions": { | |
| "assin2_rte": 1.1, | |
| "assin2_sts": 1.1, | |
| "bluex": 1.1, | |
| "enem_challenge": 1.1, | |
| "faquad_nli": 1.1, | |
| "hatebr_offensive": 1.0, | |
| "oab_exams": 1.5, | |
| "portuguese_hate_speech": 1.0, | |
| "tweetsentbr": 1.0 | |
| }, | |
| "n-shot": { | |
| "assin2_rte": 15, | |
| "assin2_sts": 10, | |
| "bluex": 3, | |
| "enem_challenge": 3, | |
| "faquad_nli": 15, | |
| "hatebr_offensive": 25, | |
| "oab_exams": 3, | |
| "portuguese_hate_speech": 25, | |
| "tweetsentbr": 25 | |
| }, | |
| "model_meta": { | |
| "truncated": 2, | |
| "non_truncated": 14148, | |
| "padded": 0, | |
| "non_padded": 14150, | |
| "fewshots_truncated": 4, | |
| "has_chat_template": false, | |
| "chat_type": null, | |
| "n_gpus": 1, | |
| "accelerate_num_process": null, | |
| "model_sha": "None", | |
| "model_dtype": "torch.float32", | |
| "model_memory_footprint": 4400229248, | |
| "model_num_parameters": 1100056576, | |
| "model_is_loaded_in_4bit": null, | |
| "model_is_loaded_in_8bit": null, | |
| "model_is_quantized": null, | |
| "model_device": "cuda:0", | |
| "batch_size": 32, | |
| "max_length": 2048, | |
| "max_ctx_length": 2016, | |
| "max_gen_toks": 32 | |
| }, | |
| "task_model_meta": { | |
| "assin2_rte": { | |
| "sample_size": 2448, | |
| "truncated": 0, | |
| "non_truncated": 2448, | |
| "padded": 0, | |
| "non_padded": 2448, | |
| "fewshots_truncated": 0, | |
| "mean_seq_length": 924.4232026143791, | |
| "min_seq_length": 909, | |
| "max_seq_length": 963, | |
| "max_ctx_length": 2016, | |
| "max_gen_toks": 32, | |
| "mean_original_fewshots_size": 15.0, | |
| "mean_effective_fewshot_size": 15.0 | |
| }, | |
| "assin2_sts": { | |
| "sample_size": 2448, | |
| "truncated": 0, | |
| "non_truncated": 2448, | |
| "padded": 0, | |
| "non_padded": 2448, | |
| "fewshots_truncated": 0, | |
| "mean_seq_length": 659.4232026143791, | |
| "min_seq_length": 644, | |
| "max_seq_length": 698, | |
| "max_ctx_length": 2016, | |
| "max_gen_toks": 32, | |
| "mean_original_fewshots_size": 10.0, | |
| "mean_effective_fewshot_size": 10.0 | |
| }, | |
| "bluex": { | |
| "sample_size": 719, | |
| "truncated": 0, | |
| "non_truncated": 719, | |
| "padded": 0, | |
| "non_padded": 719, | |
| "fewshots_truncated": 0, | |
| "mean_seq_length": 1170.817802503477, | |
| "min_seq_length": 904, | |
| "max_seq_length": 1801, | |
| "max_ctx_length": 2016, | |
| "max_gen_toks": 32, | |
| "mean_original_fewshots_size": 3.0, | |
| "mean_effective_fewshot_size": 3.0 | |
| }, | |
| "enem_challenge": { | |
| "sample_size": 1429, | |
| "truncated": 2, | |
| "non_truncated": 1427, | |
| "padded": 0, | |
| "non_padded": 1429, | |
| "fewshots_truncated": 4, | |
| "mean_seq_length": 1007.4177746675997, | |
| "min_seq_length": 829, | |
| "max_seq_length": 2484, | |
| "max_ctx_length": 2016, | |
| "max_gen_toks": 32, | |
| "mean_original_fewshots_size": 3.0, | |
| "mean_effective_fewshot_size": 2.9972008397480754 | |
| }, | |
| "faquad_nli": { | |
| "sample_size": 650, | |
| "truncated": 0, | |
| "non_truncated": 650, | |
| "padded": 0, | |
| "non_padded": 650, | |
| "fewshots_truncated": 0, | |
| "mean_seq_length": 968.1338461538462, | |
| "min_seq_length": 936, | |
| "max_seq_length": 1034, | |
| "max_ctx_length": 2016, | |
| "max_gen_toks": 32, | |
| "mean_original_fewshots_size": 15.0, | |
| "mean_effective_fewshot_size": 15.0 | |
| }, | |
| "hatebr_offensive": { | |
| "sample_size": 1400, | |
| "truncated": 0, | |
| "non_truncated": 1400, | |
| "padded": 0, | |
| "non_padded": 1400, | |
| "fewshots_truncated": 0, | |
| "mean_seq_length": 867.4407142857143, | |
| "min_seq_length": 852, | |
| "max_seq_length": 1061, | |
| "max_ctx_length": 2016, | |
| "max_gen_toks": 32, | |
| "mean_original_fewshots_size": 25.0, | |
| "mean_effective_fewshot_size": 25.0 | |
| }, | |
| "oab_exams": { | |
| "sample_size": 2195, | |
| "truncated": 0, | |
| "non_truncated": 2195, | |
| "padded": 0, | |
| "non_padded": 2195, | |
| "fewshots_truncated": 0, | |
| "mean_seq_length": 832.024145785877, | |
| "min_seq_length": 659, | |
| "max_seq_length": 1108, | |
| "max_ctx_length": 2016, | |
| "max_gen_toks": 32, | |
| "mean_original_fewshots_size": 3.0, | |
| "mean_effective_fewshot_size": 3.0 | |
| }, | |
| "portuguese_hate_speech": { | |
| "sample_size": 851, | |
| "truncated": 0, | |
| "non_truncated": 851, | |
| "padded": 0, | |
| "non_padded": 851, | |
| "fewshots_truncated": 0, | |
| "mean_seq_length": 1219.021151586369, | |
| "min_seq_length": 1192, | |
| "max_seq_length": 1255, | |
| "max_ctx_length": 2016, | |
| "max_gen_toks": 32, | |
| "mean_original_fewshots_size": 25.0, | |
| "mean_effective_fewshot_size": 25.0 | |
| }, | |
| "tweetsentbr": { | |
| "sample_size": 2010, | |
| "truncated": 0, | |
| "non_truncated": 2010, | |
| "padded": 0, | |
| "non_padded": 2010, | |
| "fewshots_truncated": 0, | |
| "mean_seq_length": 1154.4194029850746, | |
| "min_seq_length": 1137, | |
| "max_seq_length": 1211, | |
| "max_ctx_length": 2016, | |
| "max_gen_toks": 32, | |
| "mean_original_fewshots_size": 25.0, | |
| "mean_effective_fewshot_size": 25.0 | |
| } | |
| }, | |
| "config": { | |
| "model": "huggingface", | |
| "model_args": "pretrained=/lustre/mlnvme/data/asen_hpc-mula/Tucano-1b1-DPO", | |
| "batch_size": "auto", | |
| "batch_sizes": [], | |
| "device": "cuda:0", | |
| "use_cache": null, | |
| "limit": null, | |
| "bootstrap_iters": 100000, | |
| "gen_kwargs": null | |
| }, | |
| "git_hash": null | |
| } |