Add proprietary model results v1

app.py

@@ -198,6 +198,9 @@ def filter_models(
     if "Flagged" in hide_models:
         filtered_df = filtered_df[filtered_df[AutoEvalColumn.flagged.name] == False]
 
+    if "Proprietary" in hide_models:
+        filtered_df = filtered_df[filtered_df[AutoEvalColumn.license.name] != "Proprietary"]
+
     type_emoji = [t[0] for t in type_query]
     filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
     filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
@@ -231,7 +234,7 @@ leaderboard_df = filter_models(
     size_query=list(NUMERIC_INTERVALS.keys()), 
     precision_query=[i.value.name for i in Precision],
     language_query=[i.value.name for i in Language],
-    hide_models=["Contains a merge/moerge", "Flagged"], # "Private or deleted", "Contains a merge/moerge", "Flagged"
+    hide_models=["Flagged"], # "Private or deleted", "Contains a merge/moerge", "Flagged"
 )
 
 demo = gr.Blocks(css=custom_css)
@@ -268,8 +271,8 @@ with demo:
                     with gr.Row():
                         hide_models = gr.CheckboxGroup(
                             label="Hide models",
-                            choices = ["Private or deleted", "Contains a merge/moerge", "Flagged", "MoE"],
-                            value=["Contains a merge/moerge", "Flagged"],
+                            choices = ["Proprietary", "Private or deleted", "Contains a merge/moerge", "Flagged", "MoE"],
+                            value=["Flagged"],
                             interactive=True
                         )
                 with gr.Column(min_width=320):
@@ -465,7 +468,7 @@ with demo:
                     revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
                     private = gr.Checkbox(False, label="Private", visible=not IS_PUBLIC)
                     model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
+                        choices=[t.to_str(" : ") for t in ModelType if t not in [ModelType.Unknown, ModelType.proprietary]],
                         label="Model type",
                         multiselect=False,
                         value=ModelType.FT.to_str(" : "),

proprietary_models_results.json

@@ -0,0 +1,107 @@
+[
+    {
+        "model": "sabia-2-small",
+        "name": "Sabiá-2 Small",
+        "link": "https://www.maritaca.ai/",
+        "date": "2024-04-12",
+        "status": "full",
+        "main_language": "Portuguese",
+        "result_metrics": {
+            "enem_challenge": 0.7172848145556333,
+            "bluex": 0.5549374130737135,
+            "oab_exams": 0.6364464692482916,
+            "assin2_sts": 0.7053302344881672,
+            "assin2_rte": 0.9121728362223306,
+            "faquad_nli": 0.7575848453041435,
+            "hatebr_offensive": 0.5025338637870607,
+            "portuguese_hate_speech": 0.4650217578860529,
+            "tweetsentbr": 0.533977453070735
+        },
+        "result_metrics_average": 0.6428099652929031,
+        "result_metrics_npm": 0.43960062672137007
+    },
+    {
+        "model": "sabia-2-medium",
+        "name": "Sabiá-2 Medium",
+        "link": "https://www.maritaca.ai/",
+        "date": "2024-04-13",
+        "status": "full",
+        "main_language": "Portuguese",
+        "result_metrics": {
+            "enem_challenge": 0.8180545836249126,
+            "bluex": 0.717663421418637,
+            "oab_exams": 0.7321184510250569,
+            "assin2_sts": 0.7804108376537757,
+            "assin2_rte": 0.923459363368553,
+            "faquad_nli": 0.7657657657657658,
+            "hatebr_offensive": 0.8349989882997386,
+            "portuguese_hate_speech": 0.7379326358571694,
+            "tweetsentbr": 0.7269533040381798
+        },
+        "result_metrics_average": 0.7819285945613098,
+        "result_metrics_npm": 0.6676121786922709
+    },
+    {
+        "model": "gpt-3.5-turbo-0125",
+        "name": "GPT-3.5 Turbo (0125)",
+        "link": "https://www.openai.com/",
+        "date": "2024-03-08",
+        "status": "full",
+        "main_language": "English",
+        "result_metrics": {
+            "enem_challenge": 0.7214835549335199,
+            "bluex": 0.6244784422809457,
+            "oab_exams": 0.5430523917995445,
+            "assin2_sts": 0.7378460201077941,
+            "assin2_rte": 0.8823038414050672,
+            "faquad_nli": 0.746353108609074,
+            "hatebr_offensive": 0.8056205941193919,
+            "portuguese_hate_speech": 0.7363692688971499,
+            "tweetsentbr": 0.7028981330613626
+        },
+        "result_metrics_average": 0.7222672616904278,
+        "result_metrics_npm": 0.5841504766165372
+    },
+    {
+        "model": "claude-3-haiku-20240307",
+        "name": "Claude-3 Haiku (20240307)",
+        "link": "https://www.claude.ai/",
+        "date": "2024-04-13",
+        "status": "full",
+        "main_language": "English",
+        "result_metrics": {
+            "enem_challenge": 0.7718684394681595,
+            "bluex": 0.6662030598052852,
+            "oab_exams": 0.626879271070615,
+            "assin2_sts": 0.7892124744168747,
+            "assin2_rte": 0.9184462138121732,
+            "faquad_nli": 0.6340996599941455,
+            "hatebr_offensive": 0.8023698759439051,
+            "portuguese_hate_speech": 0.7342166269560177,
+            "tweetsentbr": 0.5477486799750156
+        },
+        "result_metrics_average": 0.7212271446046878,
+        "result_metrics_npm": 0.5735261536314672
+    },
+    {
+        "model": "gemini-1.0-pro",
+        "name": "Gemini 1.0 Pro",
+        "link": "https://ai.google.dev/",
+        "date": "2024-03-08",
+        "status": "full",
+        "main_language": "English",
+        "result_metrics": {
+            "enem_challenge": 0.7130860741777467,
+            "bluex": 0.5869262865090403,
+            "oab_exams": 0.4988610478359909,
+            "assin2_sts": 0.7058831239763663,
+            "assin2_rte": 0.8945993304651698,
+            "faquad_nli": 0.7070913567220611,
+            "hatebr_offensive": 0.8086330094493972,
+            "portuguese_hate_speech": 0.699119105113102,
+            "tweetsentbr": 0.6803240476660983
+        },
+        "result_metrics_average": 0.6993914868794414,
+        "result_metrics_npm": 0.551208000273598
+    }
+]

src/display/about.py

@@ -19,8 +19,6 @@ if 'readme' in TASK_CONFIG:
 INTRODUCTION_TEXT = f"""
 {GENERAL_DESCRIPTION}
 
-This is a fork of the <a href="https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard" target="_blank">🤗 Open LLM Leaderboard</a> with different benchmarks.
-
 Submit a model for automated evaluation on our GPU cluster on the "Submit" page!
 The leaderboard's backend runs on a [fork](https://github.com/eduagarcia/lm-evaluation-harness-pt) of the great [Eleuther AI Language Model Evaluation Harness](https://github.com/EleutherAI/lm-evaluation-harness) - read more details in the "About" page!  
 

src/display/utils.py

@@ -2,6 +2,9 @@ from dataclasses import dataclass, make_dataclass
 from enum import Enum
 from typing import List
 import pandas as pd
+import os
+import json
+from copy import deepcopy
 from yaml import safe_load
 from src.envs import GET_ORIGINAL_HF_LEADERBOARD_EVAL_RESULTS, TASK_CONFIG
 
@@ -87,12 +90,6 @@ baseline_row = {
     AutoEvalColumn.precision.name: "?",
     AutoEvalColumn.merged.name: False,
     #AutoEvalColumn.average.name: 31.0,
-    #AutoEvalColumn.arc.name: 25.0,
-    #AutoEvalColumn.hellaswag.name: 25.0,
-    #AutoEvalColumn.mmlu.name: 25.0,
-    #AutoEvalColumn.truthfulqa.name: 25.0,
-    #AutoEvalColumn.winogrande.name: 50.0,
-    #AutoEvalColumn.gsm8k.name: 0.21,
     AutoEvalColumn.dummy.name: "baseline",
     AutoEvalColumn.model_type.name: "",
     AutoEvalColumn.flagged.name: False,
@@ -119,8 +116,8 @@ for task in Tasks:
 baseline_row[AutoEvalColumn.average.name] = round(sum(baseline_list) / len(baseline_list), 2)
 baseline_row[AutoEvalColumn.npm.name] = round(sum(npm) / len(npm), 2)
 
-#if GET_ORIGINAL_HF_LEADERBOARD_EVAL_RESULTS:
-baseline_row["🤗 Leaderboard Average"] = None
+if GET_ORIGINAL_HF_LEADERBOARD_EVAL_RESULTS:
+    baseline_row["🤗 Leaderboard Average"] = None
 
 # Average ⬆️ human baseline is 0.897 (source: averaging human baselines below)
 # ARC human baseline is 0.80 (source: https://lab42.global/arc/)
@@ -136,12 +133,6 @@ human_baseline_row = {
     AutoEvalColumn.precision.name: "?",
     #AutoEvalColumn.average.name: 92.75,
     AutoEvalColumn.merged.name: False,
-    #AutoEvalColumn.arc.name: 80.0,
-    #AutoEvalColumn.hellaswag.name: 95.0,
-    #AutoEvalColumn.mmlu.name: 89.8,
-    #AutoEvalColumn.truthfulqa.name: 94.0,
-    #AutoEvalColumn.winogrande.name: 94.0,
-    #AutoEvalColumn.gsm8k.name: 100,
     AutoEvalColumn.dummy.name: "human_baseline",
     AutoEvalColumn.model_type.name: "",
     AutoEvalColumn.flagged.name: False,
@@ -168,8 +159,27 @@ for task in Tasks:
     npm.append((res - task.value.baseline) / (100 - task.value.baseline))
 human_baseline_row[AutoEvalColumn.average.name] = round(sum(baseline_list) / len(baseline_list), 2)
 human_baseline_row[AutoEvalColumn.npm.name] = round(sum(npm) / len(npm), 2)
-#if GET_ORIGINAL_HF_LEADERBOARD_EVAL_RESULTS:
-human_baseline_row["🤗 Leaderboard Average"] = None
+if GET_ORIGINAL_HF_LEADERBOARD_EVAL_RESULTS:
+    human_baseline_row["🤗 Leaderboard Average"] = None
+
+#Proprietary models
+proprietary_rows = []
+if os.path.exists('proprietary_models_results.json'):
+    with open('proprietary_models_results.json', 'r', encoding='utf8') as f:
+        all_models = json.load(f)
+    for model_data in all_models:
+        model_row = deepcopy(baseline_row)
+        model_row[AutoEvalColumn.model.name] = f'<a target="_blank" href="{model_data["link"]}" style="color: var(--text-color); text-decoration: underline;text-decoration-style: dotted;">{model_data["name"]} [{model_data["date"]}]</a>'
+        model_row[AutoEvalColumn.dummy.name] = model_data['model']
+        model_row[AutoEvalColumn.license.name] = "Proprietary"
+        for task in Tasks:
+            model_row[task.value.col_name] = round(model_data['result_metrics'][task.value.benchmark]*100, 2)
+        model_row[AutoEvalColumn.average.name] = round(model_data['result_metrics_average']*100, 2)
+        model_row[AutoEvalColumn.npm.name] = round(model_data['result_metrics_npm']*100, 2)
+        model_row[AutoEvalColumn.model_type.name] = "proprietary models (closed)"
+        model_row[AutoEvalColumn.model_type_symbol.name] = "🔒"
+        model_row[AutoEvalColumn.main_language.name] = model_data['main_language']
+        proprietary_rows.append(model_row)
 
 @dataclass
 class ModelDetails:
@@ -183,6 +193,7 @@ class ModelType(Enum):
     FT = ModelDetails(name="fine-tuned/fp on domain-specific datasets", symbol="🔶")
     chat = ModelDetails(name="chat models (RLHF, DPO, IFT, ...)", symbol="💬")
     merges = ModelDetails(name="base merges and moerges", symbol="🤝")
+    proprietary = ModelDetails(name="proprietary models (closed)", symbol="🔒")
     Unknown = ModelDetails(name="", symbol="?")
 
     def to_str(self, separator=" "):
@@ -200,6 +211,8 @@ class ModelType(Enum):
             return ModelType.chat
         if "merge" in type or "🤝" in type:
             return ModelType.merges
+        if "proprietary" in type or "🔒" in type:
+            return ModelType.proprietary
         return ModelType.Unknown
 
 class WeightType(Enum):
@@ -240,7 +253,7 @@ class Language(Enum):
         language = language.lower().replace('-', '').replace('_', '')
         if language in ["pt", "ptpt", "ptbr", "portuguese"]:
             return Language.Portuguese
-        if language in ["en", "enus", "engb", "english", ]:
+        if language in ["en", "enus", "engb", "english"]:
             return Language.English
         if language in ["es", "spanish"]:
             return Language.Spanish

src/populate.py

@@ -5,7 +5,7 @@ import copy
 import pandas as pd
 
 from src.display.formatting import has_no_nan_values, make_requests_clickable_model
-from src.display.utils import AutoEvalColumn, EvalQueueColumn, baseline_row
+from src.display.utils import AutoEvalColumn, EvalQueueColumn, baseline_row, proprietary_rows
 from src.leaderboard.filter_models import filter_models_flags
 from src.leaderboard.read_evals import get_raw_eval_results
 
@@ -14,6 +14,8 @@ def get_leaderboard_df(results_path: str, requests_path: str, dynamic_path: str,
     raw_data = get_raw_eval_results(results_path=results_path, requests_path=requests_path, dynamic_path=dynamic_path)
     all_data_json = [v.to_dict() for v in raw_data]
     all_data_json.append(baseline_row)
+    for proprietary_row in proprietary_rows:
+        all_data_json.append(proprietary_row)
     filter_models_flags(all_data_json)
 
     df = pd.DataFrame.from_records(all_data_json)

src/scripts/update_all_request_files.py

@@ -94,7 +94,7 @@ def update_dynamic_files():
     start = time.time()
 
     models = list(API.list_models(
-        filter=ModelFilter(task="text-generation"),
+        # filter=ModelFilter(task="text-generation"),
         full=False,
         cardData=True,
         fetch_config=True,

tasks_config/pt_config.yaml

@@ -17,18 +17,14 @@ readme:
   general_description: |
     📐 The 🚀 Open PT LLM Leaderboard aims to provide a benchmark for the evaluation of 
     Large Language Models (LLMs) in the Portuguese language across a variety of tasks 
-    and datasets.   
-    The leaderboard is open to submissions of models from the community 
-    and is designed to be a resource for researchers, practitioners, and enthusiasts 
-    interested in the development and evaluation of LLMs for the Portuguese language.  
-    If you have any questions, suggestions, or would like to contribute to the leaderboard,
-    please feel free to reach out at [@eduagarcia](https://linktr.ee/eduagarcia).
+    and datasets.     
   support_description: |
     This leaderboard is made possible by the support of the 
     [Center of Excelence in AI (CEIA)](https://ceia.ufg.br/) at the 
     [Federal University of Goiás (UFG)](https://international.ufg.br/).
 
-    Add the results to your model card: [🧐 Open Portuguese LLM Leaderboard Results PR Opener](https://huggingface.co/spaces/eduagarcia-temp/portuguese-leaderboard-results-to-modelcard)
+    If you have any questions, suggestions, or would like to contribute to the leaderboard,
+    please feel free to reach out at [@eduagarcia](https://linktr.ee/eduagarcia).
   about_description: |
     The 🚀 Open PT-LLM Leaderboard is a benchmark for the evaluation of 
     Large Language Models (LLMs) in the Portuguese language.  
@@ -41,8 +37,7 @@ readme:
     [Federal University of Goiás (UFG)](https://international.ufg.br/), this leaderboard 
     operates on a backend of Nvidia A100-80G GPUs. Evaluations are subject to 
     resource availability, which is not exclusive. Therefore, please be patient if 
-    your model is in the queue. If you'd like to support the leaderboard, feel free to 
-    reach out.
+    your model is in the queue.
 
     This is a fork of the <a href="https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard" target="_blank">🤗 Open LLM Leaderboard</a> with 
     portuguese benchmarks.