fix some bugs

.github/workflows/main.yaml

@@ -1,20 +0,0 @@
-name: Sync to Hugging Face hub
-on:
-  push:
-    branches: [main]
-
-  # to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-jobs:
-  sync-to-hub:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Push to hub
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN }}
-        run: git push https://hanhainebula:$HF_TOKEN@huggingface.co/spaces/AIR-Bench/leaderboard main

Makefile

@@ -3,21 +3,11 @@
 
 style:
 	python -m black --line-length 119 .
-	python -m black --line-length 119 src
 	python -m isort .
-	python -m isort src
 	ruff check --fix .
-	ruff check --fix src
 
 
 quality:
 	python -m black --check --line-length 119 .
-	python -m black --check --line-length 119 src
 	python -m isort --check-only .
-	python -m isort --check-only src
 	ruff check .
-	ruff check src
-
-
-test:
-	python -m pytest tests

README.md

@@ -10,14 +10,36 @@ pinned: true
 license: apache-2.0
 ---
 
-# The AIR-Bench Leaderboard repository
+# Start the configuration
 
-This repository contains the code for the AIR-Bench Leaderboard.
+Most of the variables to change for a default leaderboard are in `src/env.py` (replace the path for your leaderboard) and `src/about.py` (for tasks).
 
-| Important Links                                              | Description                                    |
-| ------------------------------------------------------------ | ---------------------------------------------- |
-| [AIR-Bench](https://github.com/AIR-Bench/AIR-Bench)          | The main repository for the AIR-Bench project. |
-| [Leaderboard Space](https://huggingface.co/spaces/AIR-Bench/leaderboard) | The leaderboard space on Hugging Face.         |
-| [Leaderboard Backend Space](https://huggingface.co/spaces/AIR-Bench/leaderboard_backend) | The leaderboard backend space on Hugging Face. |
-| [Leaderboard Code](https://github.com/AIR-Bench/leaderboard) | The code for the leaderboard.                  |
-| [Leaderboard Backend Code](https://github.com/AIR-Bench/leaderboard_backend) | The code for the leaderboard backend.          |
+Results files should have the following format and be stored as json files:
+```json
+{
+    "config": {
+        "model_dtype": "torch.float16", # or torch.bfloat16 or 8bit or 4bit
+        "model_name": "path of the model on the hub: org/model",
+        "model_sha": "revision on the hub",
+    },
+    "results": {
+        "task_name": {
+            "metric_name": score,
+        },
+        "task_name2": {
+            "metric_name": score,
+        }
+    }
+}
+```
+
+Request files are created automatically by this tool.
+
+If you encounter problem on the space, don't hesitate to restart it to remove the create eval-queue, eval-queue-bk, eval-results and eval-results-bk created folder.
+
+# Code logic for more complex edits
+
+You'll find 
+- the main table' columns names and properties in `src/display/utils.py`
+- the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
+- teh logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`

app.py

@@ -1,557 +1,391 @@
-import os
-
 import gradio as gr
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import snapshot_download
 
 from src.about import (
-    BENCHMARKS_TEXT, EVALUATION_QUEUE_TEXT, INTRODUCTION_TEXT, TITLE,
-    CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT
-)
-from src.benchmarks import LongDocBenchmarks, QABenchmarks
-from src.columns import COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL
-from src.components import (
-    get_anonymous_checkbox,
-    get_domain_dropdown,
-    get_language_dropdown,
-    get_leaderboard_table,
-    get_metric_dropdown,
-    get_noreranking_dropdown,
-    get_reranking_dropdown,
-    get_revision_and_ts_checkbox,
-    get_search_bar,
-    get_version_dropdown,
+    INTRODUCTION_TEXT,
+    BENCHMARKS_TEXT,
+    TITLE,
+    EVALUATION_QUEUE_TEXT
 )
-from src.css_html_js import custom_css
-from src.envs import (
-    API,
-    BENCHMARK_VERSION_LIST,
-    DEFAULT_METRIC_LONG_DOC,
-    DEFAULT_METRIC_QA,
-    EVAL_RESULTS_PATH,
-    LATEST_BENCHMARK_VERSION,
-    METRIC_LIST,
-    REPO_ID,
-    RESULTS_REPO,
-    TOKEN,
-)
-from src.loaders import load_eval_results
-from src.models import TaskType, model_hyperlink
-from src.utils import remove_html, reset_rank, set_listeners, submit_results, update_metric, upload_file
-
+from src.benchmarks import DOMAIN_COLS_QA, LANG_COLS_QA, DOMAIN_COLS_LONG_DOC, LANG_COLS_LONG_DOC, METRIC_LIST, \
+    DEFAULT_METRIC_QA, DEFAULT_METRIC_LONG_DOC
+from src.display.css_html_js import custom_css
+from src.display.utils import COL_NAME_IS_ANONYMOUS, COL_NAME_REVISION, COL_NAME_TIMESTAMP, COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL
+from src.envs import API, EVAL_RESULTS_PATH, REPO_ID, RESULTS_REPO, TOKEN
+from src.read_evals import get_raw_eval_results, get_leaderboard_df
+from src.utils import update_metric, upload_file, get_default_cols, submit_results, reset_rank, remove_html
+from src.display.gradio_formatting import get_version_dropdown, get_search_bar, get_reranking_dropdown, \
+    get_metric_dropdown, get_domain_dropdown, get_language_dropdown, get_anonymous_checkbox, get_revision_and_ts_checkbox, get_leaderboard_table, get_noreranking_dropdown
+from src.display.gradio_listener import set_listeners
 
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
 
 try:
-    if os.environ.get("LOCAL_MODE", False):
-        print("Loading the data")
-        snapshot_download(
-            repo_id=RESULTS_REPO,
-            local_dir=EVAL_RESULTS_PATH,
-            repo_type="dataset",
-            tqdm_class=None,
-            etag_timeout=30,
-            token=TOKEN,
-        )
-    else:
-        print("Running in local mode")
-except Exception:
-    print("failed to download")
+    snapshot_download(
+        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30,
+        token=TOKEN
+    )
+except Exception as e:
+    print(f'failed to download')
     restart_space()
 
-global ds_dict
-ds_dict = load_eval_results(EVAL_RESULTS_PATH)
-global datastore
-datastore = ds_dict[LATEST_BENCHMARK_VERSION]
-
-
-def update_qa_metric(
-    metric: str,
-    domains: list,
-    langs: list,
-    reranking_model: list,
-    query: str,
-    show_anonymous: bool,
-    show_revision_and_timestamp: bool,
-):
-    global datastore
-    return update_metric(
-        datastore,
-        TaskType.qa,
-        metric,
-        domains,
-        langs,
-        reranking_model,
-        query,
-        show_anonymous,
+raw_data = get_raw_eval_results(f"{EVAL_RESULTS_PATH}/AIR-Bench_24.04")
+
+original_df_qa = get_leaderboard_df(
+    raw_data, task='qa', metric=DEFAULT_METRIC_QA)
+original_df_long_doc = get_leaderboard_df(
+    raw_data, task='long-doc', metric=DEFAULT_METRIC_LONG_DOC)
+print(f'raw data: {len(raw_data)}')
+print(f'QA data loaded: {original_df_qa.shape}')
+print(f'Long-Doc data loaded: {len(original_df_long_doc)}')
+
+leaderboard_df_qa = original_df_qa.copy()
+# leaderboard_df_qa = leaderboard_df_qa[has_no_nan_values(df, _benchmark_cols)]
+shown_columns_qa, types_qa = get_default_cols(
+    'qa', leaderboard_df_qa.columns, add_fix_cols=True)
+leaderboard_df_qa = leaderboard_df_qa[~leaderboard_df_qa[COL_NAME_IS_ANONYMOUS]][shown_columns_qa]
+leaderboard_df_qa.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
+
+leaderboard_df_long_doc = original_df_long_doc.copy()
+shown_columns_long_doc, types_long_doc = get_default_cols(
+    'long-doc', leaderboard_df_long_doc.columns, add_fix_cols=True)
+leaderboard_df_long_doc = leaderboard_df_long_doc[~leaderboard_df_long_doc[COL_NAME_IS_ANONYMOUS]][shown_columns_long_doc]
+leaderboard_df_long_doc.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
+
+# select reranking model
+reranking_models = sorted(list(frozenset([eval_result.reranking_model for eval_result in raw_data])))
+
+
+def update_metric_qa(
+        metric: str,
+        domains: list,
+        langs: list,
+        reranking_model: list,
+        query: str,
+        show_anonymous: bool,
         show_revision_and_timestamp,
-    )
-
-
-def update_doc_metric(
-    metric: str,
-    domains: list,
-    langs: list,
-    reranking_model: list,
-    query: str,
-    show_anonymous: bool,
-    show_revision_and_timestamp,
 ):
-    global datastore
-    return update_metric(
-        datastore,
-        TaskType.long_doc,
-        metric,
-        domains,
-        langs,
-        reranking_model,
-        query,
-        show_anonymous,
+    return update_metric(raw_data, 'qa', metric, domains, langs, reranking_model, query, show_anonymous, show_revision_and_timestamp)
+
+def update_metric_long_doc(
+        metric: str,
+        domains: list,
+        langs: list,
+        reranking_model: list,
+        query: str,
+        show_anonymous: bool,
         show_revision_and_timestamp,
-    )
-
-
-def update_datastore(version):
-    global datastore
-    global ds_dict
-    if datastore.version != version:
-        print(f"updated data version: {datastore.version} -> {version}")
-        datastore = ds_dict[version]
-    else:
-        print(f"current data version: {datastore.version}")
-    return datastore
-
-
-def update_qa_domains(version):
-    datastore = update_datastore(version)
-    domain_elem = get_domain_dropdown(QABenchmarks[datastore.slug])
-    return domain_elem
-
-
-def update_doc_domains(version):
-    datastore = update_datastore(version)
-    domain_elem = get_domain_dropdown(LongDocBenchmarks[datastore.slug])
-    return domain_elem
-
-
-def update_qa_langs(version):
-    datastore = update_datastore(version)
-    lang_elem = get_language_dropdown(QABenchmarks[datastore.slug])
-    return lang_elem
-
-
-def update_doc_langs(version):
-    datastore = update_datastore(version)
-    lang_elem = get_language_dropdown(LongDocBenchmarks[datastore.slug])
-    return lang_elem
-
-
-def update_qa_models(version):
-    datastore = update_datastore(version)
-    model_elem = get_reranking_dropdown(datastore.reranking_models)
-    return model_elem
-
-
-def update_qa_df_ret_rerank(version):
-    datastore = update_datastore(version)
-    return get_leaderboard_table(datastore.qa_fmt_df, datastore.qa_types)
-
-
-def update_qa_hidden_df_ret_rerank(version):
-    datastore = update_datastore(version)
-    return get_leaderboard_table(datastore.qa_raw_df, datastore.qa_types, visible=False)
-
-
-def update_doc_df_ret_rerank(version):
-    datastore = update_datastore(version)
-    return get_leaderboard_table(datastore.doc_fmt_df, datastore.doc_types)
-
-
-def update_doc_hidden_df_ret_rerank(version):
-    datastore = update_datastore(version)
-    return get_leaderboard_table(datastore.doc_raw_df, datastore.doc_types, visible=False)
-
-
-def filter_df_ret(df):
-    df_ret = df[df[COL_NAME_RERANKING_MODEL] == "NoReranker"]
-    df_ret = reset_rank(df_ret)
-    return df_ret
-
-
-def update_qa_df_ret(version):
-    datastore = update_datastore(version)
-    df_ret = filter_df_ret(datastore.qa_fmt_df)
-    return get_leaderboard_table(df_ret, datastore.qa_types)
-
-
-def update_qa_hidden_df_ret(version):
-    datastore = update_datastore(version)
-    df_ret_hidden = filter_df_ret(datastore.qa_raw_df)
-    return get_leaderboard_table(df_ret_hidden, datastore.qa_types, visible=False)
-
-
-def update_doc_df_ret(version):
-    datastore = update_datastore(version)
-    df_ret = filter_df_ret(datastore.doc_fmt_df)
-    return get_leaderboard_table(df_ret, datastore.doc_types)
-
-
-def update_doc_hidden_df_ret(version):
-    datastore = update_datastore(version)
-    df_ret_hidden = filter_df_ret(datastore.doc_raw_df)
-    return get_leaderboard_table(df_ret_hidden, datastore.doc_types, visible=False)
-
-
-def filter_df_rerank(df):
-    df_rerank = df[df[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
-    df_rerank = reset_rank(df_rerank)
-    return df_rerank
-
-
-def update_qa_df_rerank(version):
-    datastore = update_datastore(version)
-    df_rerank = filter_df_rerank(datastore.qa_fmt_df)
-    return get_leaderboard_table(df_rerank, datastore.qa_types)
-
-
-def update_qa_hidden_df_rerank(version):
-    datastore = update_datastore(version)
-    df_rerank_hidden = filter_df_rerank(datastore.qa_raw_df)
-    return get_leaderboard_table(df_rerank_hidden, datastore.qa_types, visible=False)
-
-
-def update_doc_df_rerank(version):
-    datastore = update_datastore(version)
-    df_rerank = filter_df_rerank(datastore.doc_fmt_df)
-    return get_leaderboard_table(df_rerank, datastore.doc_types)
-
-
-def update_doc_hidden_df_rerank(version):
-    datastore = update_datastore(version)
-    df_rerank_hidden = filter_df_rerank(datastore.doc_raw_df)
-    return get_leaderboard_table(df_rerank_hidden, datastore.doc_types, visible=False)
+):
+    return update_metric(raw_data, "long-doc", metric, domains, langs, reranking_model, query, show_anonymous, show_revision_and_timestamp)
 
 
 demo = gr.Blocks(css=custom_css)
 
-BM25_LINK = model_hyperlink("https://github.com/castorini/pyserini", "BM25")
-
 with demo:
     gr.HTML(TITLE)
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("Results", elem_id="results-tab-table"):
+        with gr.TabItem("QA", elem_id="qa-benchmark-tab-table", id=0):
             with gr.Row():
-                version = get_version_dropdown()
-
-            with gr.TabItem("QA", elem_id="qa-benchmark-tab-table", id=0):
-                with gr.Row():
-                    with gr.Column(min_width=320):
-                        # select domain
-                        with gr.Row():
-                            domains = get_domain_dropdown(QABenchmarks[datastore.slug])
-                            version.change(update_qa_domains, version, domains)
-                        # select language
-                        with gr.Row():
-                            langs = get_language_dropdown(QABenchmarks[datastore.slug])
-                            version.change(update_qa_langs, version, langs)
-                    with gr.Column():
-                        # select the metric
-                        metric = get_metric_dropdown(METRIC_LIST, DEFAULT_METRIC_QA)
-                        with gr.Row():
-                            show_anonymous = get_anonymous_checkbox()
-                        with gr.Row():
-                            show_rev_ts = get_revision_and_ts_checkbox()
-                with gr.Tabs(elem_classes="tab-buttons") as sub_tabs:
-                    with gr.TabItem("Retrieval + Reranking", id=10):
-                        with gr.Row():
-                            # search retrieval models
-                            with gr.Column():
-                                search_bar = get_search_bar()
-                            # select reranking models
-                            with gr.Column():
-                                models = get_reranking_dropdown(datastore.reranking_models)
-                                version.change(update_qa_models, version, models)
-                        #  shown_table
-                        qa_df_elem_ret_rerank = get_leaderboard_table(datastore.qa_fmt_df, datastore.qa_types)
-                        version.change(update_qa_df_ret_rerank, version, qa_df_elem_ret_rerank)
-                        # Dummy leaderboard for handling the case when the user uses backspace key
-                        qa_df_elem_ret_rerank_hidden = get_leaderboard_table(
-                            datastore.qa_raw_df, datastore.qa_types, visible=False
-                        )
-                        version.change(update_qa_hidden_df_ret_rerank, version, qa_df_elem_ret_rerank_hidden)
+                with gr.Column(min_width=320):
+                    # select domain
+                    with gr.Row():
+                        selected_domains = get_domain_dropdown(DOMAIN_COLS_QA, DOMAIN_COLS_QA)
+                    # select language
+                    with gr.Row():
+                        selected_langs = get_language_dropdown(LANG_COLS_QA, LANG_COLS_QA)
+
+                with gr.Column():
+                    with gr.Row():
+                        selected_version = get_version_dropdown()
+                    # select the metric
+                    selected_metric = get_metric_dropdown(METRIC_LIST, DEFAULT_METRIC_QA)
+                    with gr.Row():
+                        show_anonymous = get_anonymous_checkbox()
+                    with gr.Row():
+                        show_revision_and_timestamp = get_revision_and_ts_checkbox()
+            with gr.Tabs(elem_classes="tab-buttons") as sub_tabs:
+                with gr.TabItem("Retrieval + Reranking", id=10):
+                    with gr.Row():
+                        # search retrieval models
+                        with gr.Column():
+                            search_bar = get_search_bar()
+                        # select reranking models
+                        with gr.Column():
+                            selected_rerankings = get_reranking_dropdown(reranking_models)
+                    leaderboard_table = get_leaderboard_table(leaderboard_df_qa, types_qa)
+                    # Dummy leaderboard for handling the case when the user uses backspace key
+                    hidden_leaderboard_table_for_search = get_leaderboard_table(original_df_qa, types_qa, visible=False)
+
+                    set_listeners(
+                        "qa",
+                        leaderboard_table,
+                        hidden_leaderboard_table_for_search,
+                        search_bar,
+                        selected_domains,
+                        selected_langs,
+                        selected_rerankings,
+                        show_anonymous,
+                        show_revision_and_timestamp,
+                    )
 
-                        set_listeners(
-                            TaskType.qa,
-                            qa_df_elem_ret_rerank,
-                            qa_df_elem_ret_rerank_hidden,
+                    # set metric listener
+                    selected_metric.change(
+                        update_metric_qa,
+                        [
+                            selected_metric,
+                            selected_domains,
+                            selected_langs,
+                            selected_rerankings,
                             search_bar,
-                            version,
-                            domains,
-                            langs,
-                            models,
                             show_anonymous,
-                            show_rev_ts,
-                        )
-
-                        # set metric listener
-                        metric.change(
-                            update_qa_metric,
-                            [metric, domains, langs, models, search_bar, show_anonymous, show_rev_ts],
-                            qa_df_elem_ret_rerank,
-                            queue=True,
-                        )
+                            show_revision_and_timestamp,
+                        ],
+                        leaderboard_table,
+                        queue=True
+                    )
+                with gr.TabItem("Retrieval Only", id=11):
+                    with gr.Row():
+                        with gr.Column(scale=1):
+                            search_bar_retriever = get_search_bar()
+                        with gr.Column(scale=1):
+                            selected_noreranker = get_noreranking_dropdown()
+                    lb_df_retriever = leaderboard_df_qa[leaderboard_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker"]
+                    lb_df_retriever = reset_rank(lb_df_retriever)
+                    lb_table_retriever = get_leaderboard_table(lb_df_retriever, types_qa)
+                    # Dummy leaderboard for handling the case when the user uses backspace key
+                    hidden_lb_df_retriever = original_df_qa[original_df_qa[COL_NAME_RERANKING_MODEL] == "NoReranker"]
+                    hidden_lb_df_retriever = reset_rank(hidden_lb_df_retriever)
+                    hidden_lb_table_retriever = get_leaderboard_table(hidden_lb_df_retriever, types_qa, visible=False)
+
+                    set_listeners(
+                        "qa",
+                        lb_table_retriever,
+                        hidden_lb_table_retriever,
+                        search_bar_retriever,
+                        selected_domains,
+                        selected_langs,
+                        selected_noreranker,
+                        show_anonymous,
+                        show_revision_and_timestamp,
+                    )
 
-                    with gr.TabItem("Retrieval Only", id=11):
-                        with gr.Row():
-                            with gr.Column(scale=1):
-                                search_bar_ret = get_search_bar()
-                            with gr.Column(scale=1):
-                                models_ret = get_noreranking_dropdown()
-                                version.change(update_qa_models, version, models_ret)
-                        _qa_df_ret = filter_df_ret(datastore.qa_fmt_df)
-                        qa_df_elem_ret = get_leaderboard_table(_qa_df_ret, datastore.qa_types)
-                        version.change(update_qa_df_ret, version, qa_df_elem_ret)
-
-                        # Dummy leaderboard for handling the case when the user uses backspace key
-                        _qa_df_ret_hidden = filter_df_ret(datastore.qa_raw_df)
-                        qa_df_elem_ret_hidden = get_leaderboard_table(
-                            _qa_df_ret_hidden, datastore.qa_types, visible=False
-                        )
-                        version.change(update_qa_hidden_df_ret, version, qa_df_elem_ret_hidden)
-
-                        set_listeners(
-                            TaskType.qa,
-                            qa_df_elem_ret,
-                            qa_df_elem_ret_hidden,
-                            search_bar_ret,
-                            version,
-                            domains,
-                            langs,
-                            models_ret,
+                    # set metric listener
+                    selected_metric.change(
+                        update_metric_qa,
+                        [
+                            selected_metric,
+                            selected_domains,
+                            selected_langs,
+                            selected_noreranker,
+                            search_bar_retriever,
                             show_anonymous,
-                            show_rev_ts,
-                        )
-
-                        metric.change(
-                            update_qa_metric,
-                            [
-                                metric,
-                                domains,
-                                langs,
-                                models_ret,
-                                search_bar_ret,
-                                show_anonymous,
-                                show_rev_ts,
-                            ],
-                            qa_df_elem_ret,
-                            queue=True,
-                        )
+                            show_revision_and_timestamp,
+                        ],
+                        lb_table_retriever,
+                        queue=True
+                    )
+                with gr.TabItem("Reranking Only", id=12):
+                    lb_df_reranker = leaderboard_df_qa[leaderboard_df_qa[COL_NAME_RETRIEVAL_MODEL] == "BM25"]
+                    lb_df_reranker = reset_rank(lb_df_reranker)
+                    reranking_models_reranker = lb_df_reranker[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
+                    with gr.Row():
+                        with gr.Column(scale=1):
+                            selected_rerankings_reranker = get_reranking_dropdown(reranking_models_reranker)
+                        with gr.Column(scale=1):
+                            search_bar_reranker = gr.Textbox(show_label=False, visible=False)
+                    lb_table_reranker = get_leaderboard_table(lb_df_reranker, types_qa)
+                    hidden_lb_df_reranker = original_df_qa[original_df_qa[COL_NAME_RETRIEVAL_MODEL] == "BM25"]
+                    hidden_lb_df_reranker = reset_rank(hidden_lb_df_reranker)
+                    hidden_lb_table_reranker = get_leaderboard_table(
+                        hidden_lb_df_reranker, types_qa, visible=False
+                    )
 
-                    with gr.TabItem("Reranking Only", id=12):
-                        _qa_df_rerank = filter_df_rerank(datastore.qa_fmt_df)
-                        qa_rerank_models = _qa_df_rerank[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
-                        with gr.Row():
-                            with gr.Column(scale=1):
-                                qa_models_rerank = get_reranking_dropdown(qa_rerank_models)
-                                version.change(update_qa_models, version, qa_models_rerank)
-                            with gr.Column(scale=1):
-                                qa_search_bar_rerank = gr.Textbox(show_label=False, visible=False)
-                        qa_df_elem_rerank = get_leaderboard_table(_qa_df_rerank, datastore.qa_types)
-                        version.change(update_qa_df_rerank, version, qa_df_elem_rerank)
-
-                        _qa_df_rerank_hidden = filter_df_rerank(datastore.qa_raw_df)
-                        qa_df_elem_rerank_hidden = get_leaderboard_table(
-                            _qa_df_rerank_hidden, datastore.qa_types, visible=False
-                        )
-                        version.change(update_qa_hidden_df_rerank, version, qa_df_elem_rerank_hidden)
-
-                        set_listeners(
-                            TaskType.qa,
-                            qa_df_elem_rerank,
-                            qa_df_elem_rerank_hidden,
-                            qa_search_bar_rerank,
-                            version,
-                            domains,
-                            langs,
-                            qa_models_rerank,
+                    set_listeners(
+                        "qa",
+                        lb_table_reranker,
+                        hidden_lb_table_reranker,
+                        search_bar_reranker,
+                        selected_domains,
+                        selected_langs,
+                        selected_rerankings_reranker,
+                        show_anonymous,
+                        show_revision_and_timestamp,
+                    )
+                    # set metric listener
+                    selected_metric.change(
+                        update_metric_qa,
+                        [
+                            selected_metric,
+                            selected_domains,
+                            selected_langs,
+                            selected_rerankings_reranker,
+                            search_bar_reranker,
                             show_anonymous,
-                            show_rev_ts,
+                            show_revision_and_timestamp,
+                        ],
+                        lb_table_reranker,
+                        queue=True
+                    )
+        with gr.TabItem("Long Doc", elem_id="long-doc-benchmark-tab-table", id=1):
+            with gr.Row():
+                with gr.Column(min_width=320):
+                    # select domain
+                    with gr.Row():
+                        selected_domains = get_domain_dropdown(DOMAIN_COLS_LONG_DOC, DOMAIN_COLS_LONG_DOC)
+                    # select language
+                    with gr.Row():
+                        selected_langs = get_language_dropdown(
+                            LANG_COLS_LONG_DOC, LANG_COLS_LONG_DOC
                         )
+                with gr.Column():
+                    with gr.Row():
+                        selected_version = get_version_dropdown()
+                    # select the metric
+                    with gr.Row():
+                        selected_metric = get_metric_dropdown(METRIC_LIST, DEFAULT_METRIC_LONG_DOC)
+                    with gr.Row():
+                        show_anonymous = get_anonymous_checkbox()
+                    with gr.Row():
+                        show_revision_and_timestamp = get_revision_and_ts_checkbox()
+            with gr.Tabs(elem_classes="tab-buttons") as sub_tabs:
+                with gr.TabItem("Retrieval + Reranking", id=20):
+                    with gr.Row():
+                        with gr.Column():
+                            search_bar = get_search_bar()
+                        # select reranking model
+                        with gr.Column():
+                            selected_rerankings = get_reranking_dropdown(reranking_models)
+
+                    lb_table = get_leaderboard_table(
+                        leaderboard_df_long_doc, types_long_doc
+                    )
 
-                        metric.change(
-                            update_qa_metric,
-                            [
-                                metric,
-                                domains,
-                                langs,
-                                qa_models_rerank,
-                                qa_search_bar_rerank,
-                                show_anonymous,
-                                show_rev_ts,
-                            ],
-                            qa_df_elem_rerank,
-                            queue=True,
-                        )
-            with gr.TabItem("Long Doc", elem_id="long-doc-benchmark-tab-table", id=1):
-                with gr.Row():
-                    with gr.Column(min_width=320):
-                        # select domain
-                        with gr.Row():
-                            domains = get_domain_dropdown(LongDocBenchmarks[datastore.slug])
-                            version.change(update_doc_domains, version, domains)
-                        # select language
-                        with gr.Row():
-                            langs = get_language_dropdown(LongDocBenchmarks[datastore.slug])
-                            version.change(update_doc_langs, version, langs)
-                    with gr.Column():
-                        # select the metric
-                        with gr.Row():
-                            metric = get_metric_dropdown(METRIC_LIST, DEFAULT_METRIC_LONG_DOC)
-                        with gr.Row():
-                            show_anonymous = get_anonymous_checkbox()
-                        with gr.Row():
-                            show_rev_ts = get_revision_and_ts_checkbox()
-                with gr.Tabs(elem_classes="tab-buttons"):
-                    with gr.TabItem("Retrieval + Reranking", id=20):
-                        with gr.Row():
-                            with gr.Column():
-                                search_bar = get_search_bar()
-                            with gr.Column():
-                                models = get_reranking_dropdown(datastore.reranking_models)
-                                version.change(update_qa_models, version, models)
-
-                        doc_df_elem_ret_rerank = get_leaderboard_table(datastore.doc_fmt_df, datastore.doc_types)
-
-                        version.change(update_doc_df_ret_rerank, version, doc_df_elem_ret_rerank)
-
-                        doc_df_elem_ret_rerank_hidden = get_leaderboard_table(
-                            datastore.doc_raw_df, datastore.doc_types, visible=False
-                        )
+                    # Dummy leaderboard for handling the case when the user uses backspace key
+                    hidden_lb_table_for_search = get_leaderboard_table(
+                        original_df_long_doc, types_long_doc, visible=False
+                    )
 
-                        version.change(update_doc_hidden_df_ret_rerank, version, doc_df_elem_ret_rerank_hidden)
+                    set_listeners(
+                        "long-doc",
+                        lb_table,
+                        hidden_lb_table_for_search,
+                        search_bar,
+                        selected_domains,
+                        selected_langs,
+                        selected_rerankings,
+                        show_anonymous,
+                        show_revision_and_timestamp,
+                    )
 
-                        set_listeners(
-                            TaskType.long_doc,
-                            doc_df_elem_ret_rerank,
-                            doc_df_elem_ret_rerank_hidden,
+                    # set metric listener
+                    selected_metric.change(
+                        update_metric_long_doc,
+                        [
+                            selected_metric,
+                            selected_domains,
+                            selected_langs,
+                            selected_rerankings,
                             search_bar,
-                            version,
-                            domains,
-                            langs,
-                            models,
                             show_anonymous,
-                            show_rev_ts,
-                        )
-
-                        # set metric listener
-                        metric.change(
-                            update_doc_metric,
-                            [
-                                metric,
-                                domains,
-                                langs,
-                                models,
-                                search_bar,
-                                show_anonymous,
-                                show_rev_ts,
-                            ],
-                            doc_df_elem_ret_rerank,
-                            queue=True,
-                        )
-                    with gr.TabItem("Retrieval Only", id=21):
-                        with gr.Row():
-                            with gr.Column(scale=1):
-                                search_bar_ret = get_search_bar()
-                            with gr.Column(scale=1):
-                                models_ret = get_noreranking_dropdown()
-                        _doc_df_ret = filter_df_ret(datastore.doc_fmt_df)
-                        doc_df_elem_ret = get_leaderboard_table(_doc_df_ret, datastore.doc_types)
-                        version.change(update_doc_df_ret, version, doc_df_elem_ret)
-
-                        _doc_df_ret_hidden = filter_df_ret(datastore.doc_raw_df)
-                        doc_df_elem_ret_hidden = get_leaderboard_table(
-                            _doc_df_ret_hidden, datastore.doc_types, visible=False
-                        )
-                        version.change(update_doc_hidden_df_ret, version, doc_df_elem_ret_hidden)
-
-                        set_listeners(
-                            TaskType.long_doc,
-                            doc_df_elem_ret,
-                            doc_df_elem_ret_hidden,
-                            search_bar_ret,
-                            version,
-                            domains,
-                            langs,
-                            models_ret,
-                            show_anonymous,
-                            show_rev_ts,
-                        )
+                            show_revision_and_timestamp
+                        ],
+                        lb_table,
+                        queue=True
+                    )
+                with gr.TabItem("Retrieval Only", id=21):
+                    with gr.Row():
+                        with gr.Column(scale=1):
+                            search_bar_retriever = get_search_bar()
+                        with gr.Column(scale=1):
+                            selected_noreranker = get_noreranking_dropdown()
+                    lb_df_retriever_long_doc = leaderboard_df_long_doc[
+                        leaderboard_df_long_doc[COL_NAME_RERANKING_MODEL] == "NoReranker"
+                    ]
+                    lb_df_retriever_long_doc = reset_rank(lb_df_retriever_long_doc)
+                    hidden_lb_db_retriever_long_doc = original_df_long_doc[
+                        original_df_long_doc[COL_NAME_RERANKING_MODEL] == "NoReranker"
+                    ]
+                    hidden_lb_db_retriever_long_doc = reset_rank(hidden_lb_db_retriever_long_doc)
+                    lb_table_retriever_long_doc = get_leaderboard_table(
+                        lb_df_retriever_long_doc, types_long_doc)
+                    hidden_lb_table_retriever_long_doc = get_leaderboard_table(
+                        hidden_lb_db_retriever_long_doc, types_long_doc, visible=False
+                    )
 
-                        metric.change(
-                            update_doc_metric,
-                            [
-                                metric,
-                                domains,
-                                langs,
-                                models_ret,
-                                search_bar_ret,
-                                show_anonymous,
-                                show_rev_ts,
-                            ],
-                            doc_df_elem_ret,
-                            queue=True,
-                        )
-                    with gr.TabItem("Reranking Only", id=22):
-                        _doc_df_rerank = filter_df_rerank(datastore.doc_fmt_df)
-                        doc_rerank_models = (
-                            _doc_df_rerank[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
-                        )
-                        with gr.Row():
-                            with gr.Column(scale=1):
-                                doc_models_rerank = get_reranking_dropdown(doc_rerank_models)
-                            with gr.Column(scale=1):
-                                doc_search_bar_rerank = gr.Textbox(show_label=False, visible=False)
-                        doc_df_elem_rerank = get_leaderboard_table(_doc_df_rerank, datastore.doc_types)
-                        version.change(update_doc_df_rerank, version, doc_df_elem_rerank)
-
-                        _doc_df_rerank_hidden = filter_df_rerank(datastore.doc_raw_df)
-                        doc_df_elem_rerank_hidden = get_leaderboard_table(
-                            _doc_df_rerank_hidden, datastore.doc_types, visible=False
-                        )
+                    set_listeners(
+                        "long-doc",
+                        lb_table_retriever_long_doc,
+                        hidden_lb_table_retriever_long_doc,
+                        search_bar_retriever,
+                        selected_domains,
+                        selected_langs,
+                        selected_noreranker,
+                        show_anonymous,
+                        show_revision_and_timestamp,
+                    )
 
-                        version.change(update_doc_hidden_df_rerank, version, doc_df_elem_rerank_hidden)
-
-                        set_listeners(
-                            TaskType.long_doc,
-                            doc_df_elem_rerank,
-                            doc_df_elem_rerank_hidden,
-                            doc_search_bar_rerank,
-                            version,
-                            domains,
-                            langs,
-                            doc_models_rerank,
+                    selected_metric.change(
+                        update_metric_long_doc,
+                        [
+                            selected_metric,
+                            selected_domains,
+                            selected_langs,
+                            selected_noreranker,
+                            search_bar_retriever,
                             show_anonymous,
-                            show_rev_ts,
-                        )
+                            show_revision_and_timestamp,
+                        ],
+                        lb_table_retriever_long_doc,
+                        queue=True
+                    )
+                with gr.TabItem("Reranking Only", id=22):
+                    lb_df_reranker_ldoc = leaderboard_df_long_doc[
+                        leaderboard_df_long_doc[COL_NAME_RETRIEVAL_MODEL] == "BM25"
+                        ]
+                    lb_df_reranker_ldoc = reset_rank(lb_df_reranker_ldoc)
+                    reranking_models_reranker_ldoc = lb_df_reranker_ldoc[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
+                    with gr.Row():
+                        with gr.Column(scale=1):
+                            selected_rerankings_reranker_ldoc = get_reranking_dropdown(reranking_models_reranker_ldoc)
+                        with gr.Column(scale=1):
+                            search_bar_reranker_ldoc = gr.Textbox(show_label=False, visible=False)
+                    lb_table_reranker_ldoc = get_leaderboard_table(lb_df_reranker_ldoc, types_long_doc)
+                    hidden_lb_df_reranker_ldoc = original_df_long_doc[original_df_long_doc[COL_NAME_RETRIEVAL_MODEL] == "BM25"]
+                    hidden_lb_df_reranker_ldoc = reset_rank(hidden_lb_df_reranker_ldoc)
+                    hidden_lb_table_reranker_ldoc = get_leaderboard_table(
+                        hidden_lb_df_reranker_ldoc, types_long_doc, visible=False
+                    )
 
-                        metric.change(
-                            update_doc_metric,
-                            [
-                                metric,
-                                domains,
-                                langs,
-                                doc_models_rerank,
-                                doc_search_bar_rerank,
-                                show_anonymous,
-                                show_rev_ts,
-                            ],
-                            doc_df_elem_rerank,
-                            queue=True,
-                        )
+                    set_listeners(
+                        "long-doc",
+                        lb_table_reranker_ldoc,
+                        hidden_lb_table_reranker_ldoc,
+                        search_bar_reranker_ldoc,
+                        selected_domains,
+                        selected_langs,
+                        selected_rerankings_reranker_ldoc,
+                        show_anonymous,
+                        show_revision_and_timestamp,
+                    )
+                    selected_metric.change(
+                        update_metric_long_doc,
+                        [
+                            selected_metric,
+                            selected_domains,
+                            selected_langs,
+                            selected_rerankings_reranker_ldoc,
+                            search_bar_reranker_ldoc,
+                            show_anonymous,
+                            show_revision_and_timestamp,
+                        ],
+                        lb_table_reranker_ldoc,
+                        queue=True
+                    )
 
         with gr.TabItem("🚀Submit here!", elem_id="submit-tab-table", id=2):
             with gr.Column():
@@ -567,18 +401,23 @@ with demo:
                 with gr.Row():
                     with gr.Column():
                         reranking_model_name = gr.Textbox(
-                            label="Reranking Model name", info="Optional", value="NoReranker"
+                            label="Reranking Model name",
+                            info="Optional",
+                            value="NoReranker"
                         )
                     with gr.Column():
-                        reranking_model_url = gr.Textbox(label="Reranking Model URL", info="Optional", value="")
+                        reranking_model_url = gr.Textbox(
+                            label="Reranking Model URL",
+                            info="Optional",
+                            value=""
+                        )
                 with gr.Row():
                     with gr.Column():
                         benchmark_version = gr.Dropdown(
-                            BENCHMARK_VERSION_LIST,
-                            value=LATEST_BENCHMARK_VERSION,
+                            ["AIR-Bench_24.04", ],
+                            value="AIR-Bench_24.04",
                             interactive=True,
-                            label="AIR-Bench Version (🟠NOTE: Select the version you want to submit to)",
-                        )
+                            label="AIR-Bench Version")
                 with gr.Row():
                     upload_button = gr.UploadButton("Click to upload search results", file_count="single")
                 with gr.Row():
@@ -587,8 +426,7 @@ with demo:
                     is_anonymous = gr.Checkbox(
                         label="Nope. I want to submit anonymously 🥷",
                         value=False,
-                        info="Do you want to shown on the leaderboard by default?",
-                    )
+                        info="Do you want to shown on the leaderboard by default?")
                 with gr.Row():
                     submit_button = gr.Button("Submit")
                 with gr.Row():
@@ -598,8 +436,7 @@ with demo:
                     [
                         upload_button,
                     ],
-                    file_output,
-                )
+                    file_output)
                 submit_button.click(
                     submit_results,
                     [
@@ -609,21 +446,16 @@ with demo:
                         reranking_model_name,
                         reranking_model_url,
                         benchmark_version,
-                        is_anonymous,
+                        is_anonymous
                     ],
                     submission_result,
-                    show_progress="hidden",
+                    show_progress="hidden"
                 )
 
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=3):
             gr.Markdown(BENCHMARKS_TEXT, elem_classes="markdown-text")
 
-    gr.Markdown(f"{CITATION_BUTTON_LABEL}\n\n{CITATION_BUTTON_TEXT}", elem_classes="markdown-text")
-
-
-if __name__ == "__main__":
-    scheduler = BackgroundScheduler()
-    scheduler.add_job(restart_space, "interval", seconds=1800)
-    scheduler.start()
-    demo.queue(default_concurrency_limit=40)
-    demo.launch()
+scheduler = BackgroundScheduler()
+scheduler.add_job(restart_space, "interval", seconds=1800)
+scheduler.start()
+demo.queue(default_concurrency_limit=40).launch()

pyproject.toml

@@ -1,9 +1,9 @@
 [tool.ruff]
 # Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
-lint.select = ["E", "F"]
-lint.ignore = ["E501"] # line too long (black is taking care of this)
+select = ["E", "F"]
+ignore = ["E501"] # line too long (black is taking care of this)
 line-length = 119
-lint.fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
+fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
 
 [tool.isort]
 profile = "black"

requirements.txt

@@ -1,15 +1,14 @@
-APScheduler>=3.10.1
-black>=23.11.0
-click>=8.1.3
-datasets>=2.14.5
-gradio<5.0.0
-gradio_client>=0.16.1
+APScheduler==3.10.1
+black==23.11.0
+click==8.1.3
+datasets==2.14.5
+gradio==4.29.0
+gradio_client==0.16.1
 huggingface-hub>=0.18.0
-numpy>=1.24.2
-pandas>=2.0.0
-python-dateutil>=2.8.2
-requests>=2.31.0
-tqdm>=4.65.0
-accelerate>=0.24.1
-socksio>=1.0.0
-air-benchmark>=0.1.0
+numpy==1.24.2
+pandas==2.0.0
+python-dateutil==2.8.2
+requests==2.31.0
+tqdm==4.65.0
+accelerate==0.24.1
+socksio==1.0.0

src/about.py

@@ -1,6 +1,6 @@
 # Your leaderboard name
 TITLE = """<h1 align="center" id="space-title">AIR-Bench: Automated Heterogeneous Information Retrieval Benchmark
- (v0.1.1) </h1>"""
+ (v0.0.3) </h1>"""
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
@@ -8,7 +8,7 @@ INTRODUCTION_TEXT = """
 """
 
 # Which evaluations are you running? how can people reproduce what you have?
-BENCHMARKS_TEXT = """
+BENCHMARKS_TEXT = f"""
 ## How the test data are generated?
 ### Find more information at [our GitHub repo](https://github.com/AIR-Bench/AIR-Bench/blob/main/docs/data_generation.md)
 
@@ -17,30 +17,14 @@ BENCHMARKS_TEXT = """
   - A: Yes, we plan to release new datasets on regular basis. However, the update frequency is to be decided.  
 
 - Q: As you are using models to do the quality control when generating the data, is it biased to the models that are used?
-  - A: Yes, the results is biased to the chosen models. However, we believe the datasets labeled by human are also biased to the human's preference. The key point to verify is whether the model's bias is consistent with the human's. We use our approach to generate test data using the well established MSMARCO datasets. We benchmark different models' performances using the generated dataset and the human-label DEV dataset. Comparing the ranking of different models on these two datasets, we observe the spearman correlation between them is 0.8211 (p-value=5e-5). This indicates that the models' perference is well aligned with the human. Please refer to [here](https://github.com/AIR-Bench/AIR-Bench/blob/main/docs/available_analysis_results.md#consistency-with-human-labeled-data) for details.
+  - A: Yes, the results is biased to the chosen models. However, we believe the datasets labeled by human are also biased to the human's preference. The key point to verify is whether the model's bias is consistent with the human's. We use our approach to generate test data using the well established MSMARCO datasets. We benchmark different models' performances using the generated dataset and the human-label DEV dataset. Comparing the ranking of different models on these two datasets, we observe the spearman correlation between them is 0.8211 (p-value=5e-5). This indicates that the models' perference is well aligned with the human. Please refer to [here](https://github.com/AIR-Bench/AIR-Bench/blob/main/docs/available_evaluation_results.md#consistency-with-ms-marco) for details
 
 """
 
 EVALUATION_QUEUE_TEXT = """
 ## Check out the submission steps at [our GitHub repo](https://github.com/AIR-Bench/AIR-Bench/blob/main/docs/submit_to_leaderboard.md)
-
-## You can find the **STATUS of Your Submission** at the [Backend Space](https://huggingface.co/spaces/AIR-Bench/leaderboard_backend)
-
-- If the status is **✔️ Success**, then you can find your results at the [Leaderboard Space](https://huggingface.co/spaces/AIR-Bench/leaderboard) in no more than one hour.
-- If the status is **❌ Failed**, please check your submission steps and try again. If you have any questions, please feel free to open an issue [here](https://github.com/AIR-Bench/AIR-Bench/issues/new).
 """
 
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
 CITATION_BUTTON_TEXT = r"""
-```bibtex
-@misc{chen2024airbench,
-      title={AIR-Bench: Automated Heterogeneous Information Retrieval Benchmark}, 
-      author={Jianlyu Chen and Nan Wang and Chaofan Li and Bo Wang and Shitao Xiao and Han Xiao and Hao Liao and Defu Lian and Zheng Liu},
-      year={2024},
-      eprint={2412.13102},
-      archivePrefix={arXiv},
-      primaryClass={cs.IR},
-      url={https://arxiv.org/abs/2412.13102}, 
-}
-```
 """

src/benchmarks.py

@@ -1,71 +1,152 @@
 from dataclasses import dataclass
 from enum import Enum
 
-from air_benchmark.tasks.tasks import BenchmarkTable
 
-from src.envs import BENCHMARK_VERSION_LIST, METRIC_LIST
-from src.models import TaskType, get_safe_name
+def get_safe_name(name: str):
+    """Get RFC 1123 compatible safe name"""
+    name = name.replace('-', '_')
+    return ''.join(
+        character.lower()
+        for character in name
+        if (character.isalnum() or character == '_'))
+
+
+dataset_dict = {
+    "qa": {
+        "wiki": {
+            "en": ["wikipedia_20240101", ],
+            "zh": ["wikipedia_20240101", ]
+        },
+        "web": {
+            "en": ["mC4", ],
+            "zh": ["mC4", ]
+        },
+        "news": {
+            "en": ["CC-News", ],
+            "zh": ["CC-News", ]
+        },
+        "healthcare": {
+            "en": ["PubMedQA", ],
+            "zh": ["Huatuo-26M", ]
+        },
+        "law": {
+            "en": ["pile-of-law", ],
+             # "zh": ["flk_npc_gov_cn", ]
+        },
+        "finance": {
+            "en": ["Reuters-Financial", ],
+            "zh": ["FinCorpus", ]
+        },
+        "arxiv": {
+            "en": ["Arxiv", ]},
+        "msmarco": {
+            "en": ["MS MARCO", ]},
+    },
+    "long-doc": {
+        "arxiv": {
+            "en": ["gpt3", "llama2", "llm-survey", "gemini"],
+        },
+        "book": {
+            "en": [
+                "origin-of-species_darwin",
+                "a-brief-history-of-time_stephen-hawking"
+            ]
+        },
+        "healthcare": {
+            "en": [
+                "pubmed_100k-200k_1",
+                "pubmed_100k-200k_2",
+                "pubmed_100k-200k_3",
+                "pubmed_40k-50k_5-merged",
+                "pubmed_30k-40k_10-merged"
+            ]
+        },
+        "law": {
+            "en": [
+                "lex_files_300k-400k",
+                "lex_files_400k-500k",
+                "lex_files_500k-600k",
+                "lex_files_600k-700k"
+            ]
+        }
+    }
+}
+
+METRIC_LIST = [
+    "ndcg_at_1",
+    "ndcg_at_3",
+    "ndcg_at_5",
+    "ndcg_at_10",
+    "ndcg_at_100",
+    "ndcg_at_1000",
+    "map_at_1",
+    "map_at_3",
+    "map_at_5",
+    "map_at_10",
+    "map_at_100",
+    "map_at_1000",
+    "recall_at_1",
+    "recall_at_3",
+    "recall_at_5",
+    "recall_at_10",
+    "recall_at_100",
+    "recall_at_1000",
+    "precision_at_1",
+    "precision_at_3",
+    "precision_at_5",
+    "precision_at_10",
+    "precision_at_100",
+    "precision_at_1000",
+    "mrr_at_1",
+    "mrr_at_3",
+    "mrr_at_5",
+    "mrr_at_10",
+    "mrr_at_100",
+    "mrr_at_1000"
+]
 
 
 @dataclass
 class Benchmark:
     name: str  # [domain]_[language]_[metric], task_key in the json file,
-    metric: str  # metric_key in the json file
+    metric: str  # ndcg_at_1 ,metric_key in the json file
     col_name: str  # [domain]_[language], name to display in the leaderboard
     domain: str
     lang: str
     task: str
 
 
-# create a function return an enum class containing all the benchmarks
-def get_qa_benchmarks_dict(version: str):
-    benchmark_dict = {}
-    for task, domain_dict in BenchmarkTable[version].items():
-        if task != TaskType.qa.value:
-            continue
-        for domain, lang_dict in domain_dict.items():
-            for lang, dataset_list in lang_dict.items():
-                benchmark_name = get_safe_name(f"{domain}_{lang}")
+qa_benchmark_dict = {}
+long_doc_benchmark_dict = {}
+for task, domain_dict in dataset_dict.items():
+    for domain, lang_dict in domain_dict.items():
+        for lang, dataset_list in lang_dict.items():
+            if task == "qa":
+                benchmark_name = f"{domain}_{lang}"
+                benchmark_name = get_safe_name(benchmark_name)
                 col_name = benchmark_name
                 for metric in dataset_list:
-                    if "test" not in dataset_list[metric]["splits"]:
-                        continue
-                    benchmark_dict[benchmark_name] = Benchmark(benchmark_name, metric, col_name, domain, lang, task)
-    return benchmark_dict
-
-
-def get_doc_benchmarks_dict(version: str):
-    benchmark_dict = {}
-    for task, domain_dict in BenchmarkTable[version].items():
-        if task != TaskType.long_doc.value:
-            continue
-        for domain, lang_dict in domain_dict.items():
-            for lang, dataset_list in lang_dict.items():
+                    qa_benchmark_dict[benchmark_name] = Benchmark(benchmark_name, metric, col_name, domain, lang, task)
+            elif task == "long-doc":
                 for dataset in dataset_list:
                     benchmark_name = f"{domain}_{lang}_{dataset}"
                     benchmark_name = get_safe_name(benchmark_name)
                     col_name = benchmark_name
-                    if "test" not in dataset_list[dataset]["splits"]:
-                        continue
                     for metric in METRIC_LIST:
-                        benchmark_dict[benchmark_name] = Benchmark(
-                            benchmark_name, metric, col_name, domain, lang, task
-                        )
-    return benchmark_dict
+                        long_doc_benchmark_dict[benchmark_name] = Benchmark(benchmark_name, metric, col_name, domain,
+                                                                            lang, task)
 
+BenchmarksQA = Enum('BenchmarksQA', qa_benchmark_dict)
+BenchmarksLongDoc = Enum('BenchmarksLongDoc', long_doc_benchmark_dict)
 
-_qa_benchmark_dict = {}
-for version in BENCHMARK_VERSION_LIST:
-    safe_version_name = get_safe_name(version)
-    _qa_benchmark_dict[safe_version_name] = Enum(f"QABenchmarks_{safe_version_name}", get_qa_benchmarks_dict(version))
+BENCHMARK_COLS_QA = [c.col_name for c in qa_benchmark_dict.values()]
+BENCHMARK_COLS_LONG_DOC = [c.col_name for c in long_doc_benchmark_dict.values()]
 
-_doc_benchmark_dict = {}
-for version in BENCHMARK_VERSION_LIST:
-    safe_version_name = get_safe_name(version)
-    _doc_benchmark_dict[safe_version_name] = Enum(
-        f"LongDocBenchmarks_{safe_version_name}", get_doc_benchmarks_dict(version)
-    )
+DOMAIN_COLS_QA = list(frozenset([c.domain for c in qa_benchmark_dict.values()]))
+LANG_COLS_QA = list(frozenset([c.lang for c in qa_benchmark_dict.values()]))
 
+DOMAIN_COLS_LONG_DOC = list(frozenset([c.domain for c in long_doc_benchmark_dict.values()]))
+LANG_COLS_LONG_DOC = list(frozenset([c.lang for c in long_doc_benchmark_dict.values()]))
 
-QABenchmarks = Enum("QABenchmarks", _qa_benchmark_dict)
-LongDocBenchmarks = Enum("LongDocBenchmarks", _doc_benchmark_dict)
+DEFAULT_METRIC_QA = "ndcg_at_10"
+DEFAULT_METRIC_LONG_DOC = "recall_at_10"

src/display/formatting.py

@@ -0,0 +1,29 @@
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def make_clickable_model(model_name: str, model_link: str):
+    # link = f"https://huggingface.co/{model_name}"
+    if not model_link or not model_link.startswith("https://") or model_name == "BM25":
+        return model_name
+    return model_hyperlink(model_link, model_name)
+
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
+
+
+def has_no_nan_values(df, columns):
+    return df[columns].notna().all(axis=1)
+
+
+def has_nan_values(df, columns):
+    return df[columns].isna().any(axis=1)

src/{components.py → display/gradio_formatting.py}

@@ -1,14 +1,12 @@
 import gradio as gr
 
-from src.envs import BENCHMARK_VERSION_LIST, LATEST_BENCHMARK_VERSION
-
 
 def get_version_dropdown():
     return gr.Dropdown(
-        choices=BENCHMARK_VERSION_LIST,
-        value=LATEST_BENCHMARK_VERSION,
+        choices=["AIR-Bench_24.04", ],
+        value="AIR-Bench_24.04",
         label="Select the version of AIR-Bench",
-        interactive=True,
+        interactive=True
     )
 
 
@@ -16,25 +14,26 @@ def get_search_bar():
     return gr.Textbox(
         placeholder=" 🔍 Search for retrieval methods (separate multiple queries with `;`) and press ENTER...",
         show_label=False,
-        info="Search the retrieval methods",
+        info="Search the retrieval methods"
     )
 
 
 def get_reranking_dropdown(model_list):
-    return gr.Dropdown(choices=model_list, label="Select the reranking models", interactive=True, multiselect=True)
+    return gr.Dropdown(
+        choices=model_list,
+        label="Select the reranking models",
+        interactive=True,
+        multiselect=True
+    )
 
 
 def get_noreranking_dropdown():
     return gr.Dropdown(
-        choices=[
-            "NoReranker",
-        ],
-        value=[
-            "NoReranker",
-        ],
+        choices=["NoReranker", ],
+        value=["NoReranker", ],
         interactive=False,
         multiselect=True,
-        visible=False,
+        visible=False
     )
 
 
@@ -53,10 +52,7 @@ def get_metric_dropdown(metric_list, default_metrics):
     )
 
 
-def get_domain_dropdown(benchmarks, default_domains=None):
-    domain_list = list(frozenset([c.value.domain for c in list(benchmarks.value)]))
-    if default_domains is None:
-        default_domains = domain_list
+def get_domain_dropdown(domain_list, default_domains):
     return gr.CheckboxGroup(
         choices=domain_list,
         value=default_domains,
@@ -65,16 +61,13 @@ def get_domain_dropdown(benchmarks, default_domains=None):
     )
 
 
-def get_language_dropdown(benchmarks, default_languages=None):
-    language_list = list(frozenset([c.value.lang for c in list(benchmarks.value)]))
-    if default_languages is None:
-        default_languages = language_list
+def get_language_dropdown(language_list, default_languages):
     return gr.Dropdown(
         choices=language_list,
-        value=default_languages,
+        value=language_list,
         label="Select the languages",
         multiselect=True,
-        interactive=True,
+        interactive=True
     )
 
 
@@ -82,13 +75,15 @@ def get_anonymous_checkbox():
     return gr.Checkbox(
         label="Show anonymous submissions",
         value=False,
-        info="The anonymous submissions might have invalid model information.",
+        info="The anonymous submissions might have invalid model information."
     )
 
 
 def get_revision_and_ts_checkbox():
     return gr.Checkbox(
-        label="Show submission details", value=False, info="Show the revision and timestamp information of submissions"
+        label="Show submission details",
+        value=False,
+        info="Show the revision and timestamp information of submissions"
     )
 
 

src/display/gradio_listener.py

@@ -0,0 +1,53 @@
+from src.utils import update_table, update_table_long_doc
+
+
+def set_listeners(
+        task,
+        displayed_leaderboard,
+        hidden_leaderboard,
+        search_bar,
+        selected_domains,
+        selected_langs,
+        selected_rerankings,
+        show_anonymous,
+        show_revision_and_timestamp,
+
+):
+    if task == "qa":
+        update_table_func = update_table
+    elif task == "long-doc":
+        update_table_func = update_table_long_doc
+    else:
+        raise NotImplementedError
+    # Set search_bar listener
+    search_bar.submit(
+        update_table_func,
+        [
+            hidden_leaderboard,  #  hidden_leaderboard_table_for_search,
+            selected_domains,
+            selected_langs,
+            selected_rerankings,
+            search_bar,
+            show_anonymous,
+        ],
+        displayed_leaderboard
+    )
+
+    # Set column-wise listener
+    for selector in [
+        selected_domains, selected_langs, show_anonymous, show_revision_and_timestamp, selected_rerankings
+    ]:
+        selector.change(
+            update_table_func,
+            [
+                hidden_leaderboard,
+                selected_domains,
+                selected_langs,
+                selected_rerankings,
+                search_bar,
+                show_anonymous,
+                show_revision_and_timestamp
+            ],
+            displayed_leaderboard,
+            queue=True,
+        )

src/{columns.py → display/utils.py}

@@ -1,7 +1,9 @@
 from dataclasses import dataclass, make_dataclass
 
+from src.benchmarks import BenchmarksQA, BenchmarksLongDoc
 
-def _fields(raw_class):
+
+def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
 
 
@@ -17,22 +19,28 @@ class ColumnContent:
     never_hidden: bool = False
 
 
+COL_NAME_AVG = "Average ⬆️"
+COL_NAME_RETRIEVAL_MODEL = "Retrieval Method"
+COL_NAME_RERANKING_MODEL = "Reranking Model"
+COL_NAME_RETRIEVAL_MODEL_LINK = "Retrieval Model LINK"
+COL_NAME_RERANKING_MODEL_LINK = "Reranking Model LINK"
+COL_NAME_RANK = "Rank 🏆"
+COL_NAME_REVISION = "Revision"
+COL_NAME_TIMESTAMP = "Submission Date"
+COL_NAME_IS_ANONYMOUS = "Anonymous Submission"
+
+
 def get_default_auto_eval_column_dict():
     auto_eval_column_dict = []
-    auto_eval_column_dict.append(["rank", ColumnContent, ColumnContent(COL_NAME_RANK, "number", True)])
+    # Init
     auto_eval_column_dict.append(
-        [
-            "retrieval_model",
-            ColumnContent,
-            ColumnContent(COL_NAME_RETRIEVAL_MODEL, "markdown", True, never_hidden=True),
-        ]
+        ["rank", ColumnContent, ColumnContent(COL_NAME_RANK, "number", True)]
     )
     auto_eval_column_dict.append(
-        [
-            "reranking_model",
-            ColumnContent,
-            ColumnContent(COL_NAME_RERANKING_MODEL, "markdown", True, never_hidden=True),
-        ]
+        ["retrieval_model", ColumnContent, ColumnContent(COL_NAME_RETRIEVAL_MODEL, "markdown", True, hidden=False, never_hidden=True)]
+    )
+    auto_eval_column_dict.append(
+        ["reranking_model", ColumnContent, ColumnContent(COL_NAME_RERANKING_MODEL, "markdown", True, hidden=False, never_hidden=True)]
     )
     auto_eval_column_dict.append(
         ["revision", ColumnContent, ColumnContent(COL_NAME_REVISION, "markdown", True, never_hidden=True)]
@@ -40,30 +48,14 @@ def get_default_auto_eval_column_dict():
     auto_eval_column_dict.append(
         ["timestamp", ColumnContent, ColumnContent(COL_NAME_TIMESTAMP, "date", True, never_hidden=True)]
     )
-    auto_eval_column_dict.append(["average", ColumnContent, ColumnContent(COL_NAME_AVG, "number", True)])
     auto_eval_column_dict.append(
-        [
-            "retrieval_model_link",
-            ColumnContent,
-            ColumnContent(
-                COL_NAME_RETRIEVAL_MODEL_LINK,
-                "markdown",
-                False,
-                hidden=True,
-            ),
-        ]
+        ["average", ColumnContent, ColumnContent(COL_NAME_AVG, "number", True)]
+    )
+    auto_eval_column_dict.append(
+        ["retrieval_model_link", ColumnContent, ColumnContent(COL_NAME_RETRIEVAL_MODEL_LINK, "markdown", False, hidden=True, never_hidden=False)]
     )
     auto_eval_column_dict.append(
-        [
-            "reranking_model_link",
-            ColumnContent,
-            ColumnContent(
-                COL_NAME_RERANKING_MODEL_LINK,
-                "markdown",
-                False,
-                hidden=True,
-            ),
-        ]
+        ["reranking_model_link", ColumnContent, ColumnContent(COL_NAME_RERANKING_MODEL_LINK, "markdown", False, hidden=True, never_hidden=False)]
     )
     auto_eval_column_dict.append(
         ["is_anonymous", ColumnContent, ColumnContent(COL_NAME_IS_ANONYMOUS, "bool", False, hidden=True)]
@@ -71,10 +63,10 @@ def get_default_auto_eval_column_dict():
     return auto_eval_column_dict
 
 
-def make_autoevalcolumn(cls_name, benchmarks):
+def make_autoevalcolumn(cls_name="BenchmarksQA", benchmarks=BenchmarksQA):
     auto_eval_column_dict = get_default_auto_eval_column_dict()
-    # Leaderboard columns
-    for benchmark in list(benchmarks.value):
+    ## Leaderboard columns
+    for benchmark in benchmarks:
         auto_eval_column_dict.append(
             [benchmark.name, ColumnContent, ColumnContent(benchmark.value.col_name, "number", True)]
         )
@@ -83,24 +75,19 @@ def make_autoevalcolumn(cls_name, benchmarks):
     return make_dataclass(cls_name, auto_eval_column_dict, frozen=True)
 
 
-def get_default_col_names_and_types(benchmarks):
-    AutoEvalColumn = make_autoevalcolumn("AutoEvalColumn", benchmarks)
-    col_names = [c.name for c in _fields(AutoEvalColumn) if not c.hidden]
-    col_types = [c.type for c in _fields(AutoEvalColumn) if not c.hidden]
-    return col_names, col_types
+AutoEvalColumnQA = make_autoevalcolumn(
+    "AutoEvalColumnQA", BenchmarksQA)
+AutoEvalColumnLongDoc = make_autoevalcolumn(
+    "AutoEvalColumnLongDoc", BenchmarksLongDoc)
 
 
-def get_fixed_col_names_and_types():
-    fixed_cols = get_default_auto_eval_column_dict()[:-3]
-    return [c.name for _, _, c in fixed_cols], [c.type for _, _, c in fixed_cols]
+# Column selection
+COLS_QA = [c.name for c in fields(AutoEvalColumnQA) if not c.hidden]
+COLS_LONG_DOC = [c.name for c in fields(AutoEvalColumnLongDoc) if not c.hidden]
+TYPES_QA = [c.type for c in fields(AutoEvalColumnQA) if not c.hidden]
+TYPES_LONG_DOC = [c.type for c in fields(AutoEvalColumnLongDoc) if not c.hidden]
+COLS_LITE = [c.name for c in fields(AutoEvalColumnQA) if c.displayed_by_default and not c.hidden]
 
+QA_BENCHMARK_COLS = [t.value.col_name for t in BenchmarksQA]
 
-COL_NAME_AVG = "Average ⬆️"
-COL_NAME_RETRIEVAL_MODEL = "Retrieval Method"
-COL_NAME_RERANKING_MODEL = "Reranking Model"
-COL_NAME_RETRIEVAL_MODEL_LINK = "Retrieval Model LINK"
-COL_NAME_RERANKING_MODEL_LINK = "Reranking Model LINK"
-COL_NAME_RANK = "Rank 🏆"
-COL_NAME_REVISION = "Revision"
-COL_NAME_TIMESTAMP = "Submission Date"
-COL_NAME_IS_ANONYMOUS = "Anonymous Submission"
+LONG_DOC_BENCHMARK_COLS = [t.value.col_name for t in BenchmarksLongDoc]

src/envs.py

@@ -6,9 +6,7 @@ from huggingface_hub import HfApi
 # ----------------------------------
 TOKEN = os.environ.get("TOKEN", "")  # A read/write token for your org
 
-OWNER = (
-    "AIR-Bench"  # Change to your org - don't forget to create a results and request dataset, with the correct format!
-)
+OWNER = "AIR-Bench"  # "nan"  # Change to your org - don't forget to create a results and request dataset, with the correct format!
 # ----------------------------------
 
 REPO_ID = f"{OWNER}/leaderboard"
@@ -17,51 +15,10 @@ RESULTS_REPO = f"{OWNER}/eval_results"
 # repo for submitting the evaluation
 SEARCH_RESULTS_REPO = f"{OWNER}/search_results"
 
-# If you set up a cache later, just change HF_HOME
+# If you setup a cache later, just change HF_HOME
 CACHE_PATH = os.getenv("HF_HOME", ".")
 
 # Local caches
 EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval_results")
 
 API = HfApi(token=TOKEN)
-
-BENCHMARK_VERSION_LIST = [
-    "AIR-Bench_24.04",
-    "AIR-Bench_24.05",
-]
-
-LATEST_BENCHMARK_VERSION = BENCHMARK_VERSION_LIST[-1]   # Change to the latest benchmark version
-DEFAULT_METRIC_QA = "ndcg_at_10"
-DEFAULT_METRIC_LONG_DOC = "recall_at_10"
-METRIC_LIST = [
-    "ndcg_at_1",
-    "ndcg_at_3",
-    "ndcg_at_5",
-    "ndcg_at_10",
-    "ndcg_at_100",
-    "ndcg_at_1000",
-    "map_at_1",
-    "map_at_3",
-    "map_at_5",
-    "map_at_10",
-    "map_at_100",
-    "map_at_1000",
-    "recall_at_1",
-    "recall_at_3",
-    "recall_at_5",
-    "recall_at_10",
-    "recall_at_100",
-    "recall_at_1000",
-    "precision_at_1",
-    "precision_at_3",
-    "precision_at_5",
-    "precision_at_10",
-    "precision_at_100",
-    "precision_at_1000",
-    "mrr_at_1",
-    "mrr_at_3",
-    "mrr_at_5",
-    "mrr_at_10",
-    "mrr_at_100",
-    "mrr_at_1000",
-]

src/loaders.py

@@ -1,88 +0,0 @@
-import os.path
-from pathlib import Path
-from typing import Dict, List, Union
-
-import pandas as pd
-
-from src.columns import COL_NAME_IS_ANONYMOUS, COL_NAME_REVISION, COL_NAME_TIMESTAMP
-from src.envs import BENCHMARK_VERSION_LIST, DEFAULT_METRIC_LONG_DOC, DEFAULT_METRIC_QA
-from src.models import FullEvalResult, LeaderboardDataStore, TaskType, get_safe_name
-from src.utils import get_default_cols, get_leaderboard_df, reset_rank
-
-pd.options.mode.copy_on_write = True
-
-
-def load_raw_eval_results(results_path: Union[Path, str]) -> List[FullEvalResult]:
-    """
-    Load the evaluation results from a json file
-    """
-    model_result_filepaths = []
-    for root, dirs, files in os.walk(results_path):
-        if len(files) == 0:
-            continue
-
-        # select the latest results
-        for file in files:
-            if not (file.startswith("results") and file.endswith(".json")):
-                print(f"skip {file}")
-                continue
-            model_result_filepaths.append(os.path.join(root, file))
-
-    eval_results = {}
-    for model_result_filepath in model_result_filepaths:
-        # create evaluation results
-        try:
-            eval_result = FullEvalResult.init_from_json_file(model_result_filepath)
-        except UnicodeDecodeError:
-            print(f"loading file failed. {model_result_filepath}")
-            continue
-        print(f"file loaded: {model_result_filepath}")
-        timestamp = eval_result.timestamp
-        eval_results[timestamp] = eval_result
-
-    results = []
-    for k, v in eval_results.items():
-        try:
-            v.to_dict()
-            results.append(v)
-        except KeyError:
-            print(f"loading failed: {k}")
-            continue
-    return results
-
-
-def load_leaderboard_datastore(file_path, version) -> LeaderboardDataStore:
-    ds = LeaderboardDataStore(version, get_safe_name(version))
-    ds.raw_data = load_raw_eval_results(file_path)
-    print(f"raw data: {len(ds.raw_data)}")
-
-    ds.qa_raw_df = get_leaderboard_df(ds, TaskType.qa, DEFAULT_METRIC_QA)
-    print(f"QA data loaded: {ds.qa_raw_df.shape}")
-    ds.qa_fmt_df = ds.qa_raw_df.copy()
-    qa_cols, ds.qa_types = get_default_cols(TaskType.qa, ds.slug, add_fix_cols=True)
-    # by default, drop the anonymous submissions
-    ds.qa_fmt_df = ds.qa_fmt_df[~ds.qa_fmt_df[COL_NAME_IS_ANONYMOUS]][qa_cols]
-    # reset the rank after dropping the anonymous submissions
-    ds.qa_fmt_df = reset_rank(ds.qa_fmt_df)
-    ds.qa_fmt_df.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
-
-    ds.doc_raw_df = get_leaderboard_df(ds, TaskType.long_doc, DEFAULT_METRIC_LONG_DOC)
-    print(f"Long-Doc data loaded: {len(ds.doc_raw_df)}")
-    ds.doc_fmt_df = ds.doc_raw_df.copy()
-    doc_cols, ds.doc_types = get_default_cols(TaskType.long_doc, ds.slug, add_fix_cols=True)
-    # by default, drop the anonymous submissions
-    ds.doc_fmt_df = ds.doc_fmt_df[~ds.doc_fmt_df[COL_NAME_IS_ANONYMOUS]][doc_cols]
-    # reset the rank after dropping the anonymous submissions
-    ds.doc_fmt_df = reset_rank(ds.doc_fmt_df)
-    ds.doc_fmt_df.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
-
-    ds.reranking_models = sorted(list(frozenset([eval_result.reranking_model for eval_result in ds.raw_data])))
-    return ds
-
-
-def load_eval_results(file_path: Union[str, Path]) -> Dict[str, LeaderboardDataStore]:
-    output = {}
-    for version in BENCHMARK_VERSION_LIST:
-        fn = f"{file_path}/{version}"
-        output[version] = load_leaderboard_datastore(fn, version)
-    return output

src/{models.py → read_evals.py}

@@ -1,21 +1,38 @@
 import json
+import os.path
 from collections import defaultdict
 from dataclasses import dataclass
-from enum import Enum
 from typing import List
 
 import pandas as pd
 
-from src.columns import (
-    COL_NAME_IS_ANONYMOUS,
+from src.benchmarks import get_safe_name
+from src.display.utils import (
     COL_NAME_RERANKING_MODEL,
-    COL_NAME_RERANKING_MODEL_LINK,
     COL_NAME_RETRIEVAL_MODEL,
+    COL_NAME_RERANKING_MODEL_LINK,
     COL_NAME_RETRIEVAL_MODEL_LINK,
     COL_NAME_REVISION,
     COL_NAME_TIMESTAMP,
+    COL_NAME_IS_ANONYMOUS,
+    COLS_QA,
+    QA_BENCHMARK_COLS,
+    COLS_LONG_DOC,
+    LONG_DOC_BENCHMARK_COLS,
+    COL_NAME_AVG,
+    COL_NAME_RANK
 )
 
+from src.display.formatting import make_clickable_model
+
+pd.options.mode.copy_on_write = True
+
+def calculate_mean(row):
+    if pd.isna(row).any():
+        return 0
+    else:
+        return row.mean()
+
 
 @dataclass
 class EvalResult:
@@ -23,7 +40,6 @@ class EvalResult:
     Evaluation result of a single embedding model with a specific reranking model on benchmarks over different
     domains, languages, and datasets
     """
-
     eval_name: str  # name of the evaluation, [retrieval_model]_[reranking_model]_[metric]
     retrieval_model: str
     reranking_model: str
@@ -40,7 +56,6 @@ class FullEvalResult:
     """
     Evaluation result of a single embedding model with a specific reranking model on benchmarks over different tasks
     """
-
     eval_name: str  # name of the evaluation, [retrieval_model]_[reranking_model]
     retrieval_model: str
     reranking_model: str
@@ -64,6 +79,7 @@ class FullEvalResult:
         result_list = []
         retrieval_model_link = ""
         reranking_model_link = ""
+        revision = ""
         for item in model_data:
             config = item.get("config", {})
             # eval results for different metrics
@@ -82,26 +98,24 @@ class FullEvalResult:
                 metric=config["metric"],
                 timestamp=config.get("timestamp", "2024-05-12T12:24:02Z"),
                 revision=config.get("revision", "3a2ba9dcad796a48a02ca1147557724e"),
-                is_anonymous=config.get("is_anonymous", False),
+                is_anonymous=config.get("is_anonymous", False)
             )
             result_list.append(eval_result)
-        eval_result = result_list[0]
         return cls(
-            eval_name=f"{eval_result.retrieval_model}_{eval_result.reranking_model}",
-            retrieval_model=eval_result.retrieval_model,
-            reranking_model=eval_result.reranking_model,
+            eval_name=f"{result_list[0].retrieval_model}_{result_list[0].reranking_model}",
+            retrieval_model=result_list[0].retrieval_model,
+            reranking_model=result_list[0].reranking_model,
             retrieval_model_link=retrieval_model_link,
             reranking_model_link=reranking_model_link,
             results=result_list,
-            timestamp=eval_result.timestamp,
-            revision=eval_result.revision,
-            is_anonymous=eval_result.is_anonymous,
+            timestamp=result_list[0].timestamp,
+            revision=result_list[0].revision,
+            is_anonymous=result_list[0].is_anonymous
         )
 
-    def to_dict(self, task="qa", metric="ndcg_at_3") -> List:
+    def to_dict(self, task='qa', metric='ndcg_at_3') -> List:
         """
-        Convert the results in all the EvalResults over different tasks and metrics.
-        The output is a list of dict compatible with the dataframe UI
+        Convert the results in all the EvalResults over different tasks and metrics. The output is a list of dict compatible with the dataframe UI
         """
         results = defaultdict(dict)
         for eval_result in self.results:
@@ -109,66 +123,103 @@ class FullEvalResult:
                 continue
             if eval_result.task != task:
                 continue
-            eval_name = eval_result.eval_name
-            results[eval_name]["eval_name"] = eval_name
-            results[eval_name][COL_NAME_RETRIEVAL_MODEL] = make_clickable_model(
-                self.retrieval_model, self.retrieval_model_link
-            )
-            results[eval_name][COL_NAME_RERANKING_MODEL] = make_clickable_model(
-                self.reranking_model, self.reranking_model_link
-            )
-            results[eval_name][COL_NAME_RETRIEVAL_MODEL_LINK] = self.retrieval_model_link
-            results[eval_name][COL_NAME_RERANKING_MODEL_LINK] = self.reranking_model_link
-            results[eval_name][COL_NAME_REVISION] = self.revision
-            results[eval_name][COL_NAME_TIMESTAMP] = self.timestamp
-            results[eval_name][COL_NAME_IS_ANONYMOUS] = self.is_anonymous
-
+            results[eval_result.eval_name]["eval_name"] = eval_result.eval_name
+            results[eval_result.eval_name][COL_NAME_RETRIEVAL_MODEL] = (
+                make_clickable_model(self.retrieval_model, self.retrieval_model_link))
+            results[eval_result.eval_name][COL_NAME_RERANKING_MODEL] = (
+                make_clickable_model(self.reranking_model, self.reranking_model_link))
+            results[eval_result.eval_name][COL_NAME_RETRIEVAL_MODEL_LINK] = self.retrieval_model_link
+            results[eval_result.eval_name][COL_NAME_RERANKING_MODEL_LINK] = self.reranking_model_link
+            results[eval_result.eval_name][COL_NAME_REVISION] = self.revision
+            results[eval_result.eval_name][COL_NAME_TIMESTAMP] = self.timestamp
+            results[eval_result.eval_name][COL_NAME_IS_ANONYMOUS] = self.is_anonymous
+
+            # print(f'result loaded: {eval_result.eval_name}')
             for result in eval_result.results:
                 # add result for each domain, language, and dataset
                 domain = result["domain"]
                 lang = result["lang"]
                 dataset = result["dataset"]
                 value = result["value"] * 100
-                if dataset == "default":
+                if dataset == 'default':
                     benchmark_name = f"{domain}_{lang}"
                 else:
                     benchmark_name = f"{domain}_{lang}_{dataset}"
-                results[eval_name][get_safe_name(benchmark_name)] = value
+                results[eval_result.eval_name][get_safe_name(benchmark_name)] = value
         return [v for v in results.values()]
 
 
-@dataclass
-class LeaderboardDataStore:
-    version: str
-    slug: str
-    raw_data: list = None
-    qa_raw_df: pd.DataFrame = pd.DataFrame()
-    doc_raw_df: pd.DataFrame = pd.DataFrame()
-    qa_fmt_df: pd.DataFrame = pd.DataFrame()
-    doc_fmt_df: pd.DataFrame = pd.DataFrame()
-    reranking_models: list = None
-    qa_types: list = None
-    doc_types: list = None
-
-
-# Define an enum class with the name `TaskType`. There are two types of tasks, `qa` and `long-doc`.
-class TaskType(Enum):
-    qa = "qa"
-    long_doc = "long-doc"
-
-
-def make_clickable_model(model_name: str, model_link: str):
-    # link = f"https://huggingface.co/{model_name}"
-    if not model_link or not model_link.startswith("https://"):
-        return model_name
-    return model_hyperlink(model_link, model_name)
-
-
-def model_hyperlink(link, model_name):
-    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-
-
-def get_safe_name(name: str):
-    """Get RFC 1123 compatible safe name"""
-    name = name.replace("-", "_")
-    return "".join(character.lower() for character in name if (character.isalnum() or character == "_"))
+def get_raw_eval_results(results_path: str) -> List[FullEvalResult]:
+    """
+    Load the evaluation results from a json file
+    """
+    model_result_filepaths = []
+    for root, dirs, files in os.walk(results_path):
+        if len(files) == 0:
+            continue
+
+        # select the latest results
+        for file in files:
+            if not (file.startswith("results") and file.endswith(".json")):
+                print(f'skip {file}')
+                continue
+            model_result_filepaths.append(os.path.join(root, file))
+
+    eval_results = {}
+    for model_result_filepath in model_result_filepaths:
+        # create evaluation results
+        try:
+            eval_result = FullEvalResult.init_from_json_file(model_result_filepath)
+        except UnicodeDecodeError as e:
+            print(f"loading file failed. {model_result_filepath}")
+            continue
+        print(f'file loaded: {model_result_filepath}')
+        eval_name = eval_result.eval_name
+        eval_results[eval_name] = eval_result
+
+    results = []
+    for k, v in eval_results.items():
+        try:
+            v.to_dict()
+            results.append(v)
+        except KeyError:
+            print(f"loading failed: {k}")
+            continue
+    return results
+
+
+def get_leaderboard_df(raw_data: List[FullEvalResult], task: str, metric: str) -> pd.DataFrame:
+    """
+    Creates a dataframe from all the individual experiment results
+    """
+    cols = [COL_NAME_IS_ANONYMOUS, ]
+    if task == "qa":
+        cols += COLS_QA
+        benchmark_cols = QA_BENCHMARK_COLS
+    elif task == "long-doc":
+        cols += COLS_LONG_DOC
+        benchmark_cols = LONG_DOC_BENCHMARK_COLS
+    else:
+        raise NotImplemented
+    all_data_json = []
+    for v in raw_data:
+        all_data_json += v.to_dict(task=task, metric=metric)
+    df = pd.DataFrame.from_records(all_data_json)
+    # print(f'dataframe created: {df.shape}')
+
+    _benchmark_cols = frozenset(benchmark_cols).intersection(frozenset(df.columns.to_list()))
+
+    # calculate the average score for selected benchmarks
+    df[COL_NAME_AVG] = df[list(_benchmark_cols)].apply(calculate_mean, axis=1).round(decimals=2)
+    df.sort_values(by=[COL_NAME_AVG], ascending=False, inplace=True)
+    df.reset_index(inplace=True, drop=True)
+
+    _cols = frozenset(cols).intersection(frozenset(df.columns.to_list()))
+    df = df[_cols].round(decimals=2)
+
+    # filter out if any of the benchmarks have not been produced
+    df[COL_NAME_RANK] = df[COL_NAME_AVG].rank(ascending=False, method="min")
+
+    # shorten the revision
+    df[COL_NAME_REVISION] = df[COL_NAME_REVISION].str[:6]
+    return df

src/utils.py

@@ -1,37 +1,24 @@
-import hashlib
 import json
-import re
+import hashlib
 from datetime import datetime, timezone
 from pathlib import Path
+from typing import List
 
 import pandas as pd
 
-from src.benchmarks import LongDocBenchmarks, QABenchmarks
-from src.columns import (
-    COL_NAME_AVG,
-    COL_NAME_IS_ANONYMOUS,
-    COL_NAME_RANK,
-    COL_NAME_RERANKING_MODEL,
-    COL_NAME_RETRIEVAL_MODEL,
-    COL_NAME_REVISION,
-    COL_NAME_TIMESTAMP,
-    get_default_col_names_and_types,
-    get_fixed_col_names_and_types,
-)
-from src.envs import API, LATEST_BENCHMARK_VERSION, SEARCH_RESULTS_REPO
-from src.models import TaskType, get_safe_name
-
-
-def calculate_mean(row):
-    if pd.isna(row).any():
-        return -1
-    else:
-        return row.mean()
+from src.benchmarks import BENCHMARK_COLS_QA, BENCHMARK_COLS_LONG_DOC, BenchmarksQA, BenchmarksLongDoc
+from src.display.formatting import styled_message, styled_error
+from src.display.utils import COLS_QA, TYPES_QA, COLS_LONG_DOC, TYPES_LONG_DOC, COL_NAME_RANK, COL_NAME_AVG, \
+    COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL, COL_NAME_IS_ANONYMOUS, COL_NAME_TIMESTAMP, COL_NAME_REVISION, get_default_auto_eval_column_dict
+from src.envs import API, SEARCH_RESULTS_REPO
+from src.read_evals import FullEvalResult, get_leaderboard_df, calculate_mean
+
+import re
 
 
 def remove_html(input_str):
     # Regular expression for finding HTML tags
-    clean = re.sub(r"<.*?>", "", input_str)
+    clean = re.sub(r'<.*?>', '', input_str)
     return clean
 
 
@@ -68,152 +55,160 @@ def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
     return df[(df[COL_NAME_RETRIEVAL_MODEL].str.contains(query, case=False))]
 
 
-def get_default_cols(task: TaskType, version_slug, add_fix_cols: bool = True) -> tuple:
+def get_default_cols(task: str, columns: list=[], add_fix_cols: bool=True) -> list:
     cols = []
     types = []
-    if task == TaskType.qa:
-        benchmarks = QABenchmarks[version_slug]
-    elif task == TaskType.long_doc:
-        benchmarks = LongDocBenchmarks[version_slug]
+    if task == "qa":
+        cols_list = COLS_QA
+        types_list = TYPES_QA
+        benchmark_list = BENCHMARK_COLS_QA
+    elif task == "long-doc":
+        cols_list = COLS_LONG_DOC
+        types_list = TYPES_LONG_DOC
+        benchmark_list = BENCHMARK_COLS_LONG_DOC
     else:
-        raise NotImplementedError
-    cols_list, types_list = get_default_col_names_and_types(benchmarks)
-    benchmark_list = [c.value.col_name for c in list(benchmarks.value)]
+        raise NotImplemented
     for col_name, col_type in zip(cols_list, types_list):
         if col_name not in benchmark_list:
             continue
+        if len(columns) > 0 and col_name not in columns:
+            continue
         cols.append(col_name)
         types.append(col_type)
+
     if add_fix_cols:
         _cols = []
         _types = []
-        fixed_cols, fixed_cols_types = get_fixed_col_names_and_types()
         for col_name, col_type in zip(cols, types):
-            if col_name in fixed_cols:
+            if col_name in FIXED_COLS:
                 continue
             _cols.append(col_name)
             _types.append(col_type)
-        cols = fixed_cols + _cols
-        types = fixed_cols_types + _types
+        cols = FIXED_COLS + _cols
+        types = FIXED_COLS_TYPES + _types
     return cols, types
 
 
-def get_selected_cols(task, version_slug, domains, languages):
-    cols, _ = get_default_cols(task=task, version_slug=version_slug, add_fix_cols=False)
+fixed_cols = get_default_auto_eval_column_dict()[:-3]
+
+FIXED_COLS = [c.name for _, _, c in fixed_cols]
+FIXED_COLS_TYPES = [c.type for _, _, c in fixed_cols]
+
+
+def select_columns(
+        df: pd.DataFrame,
+        domain_query: list,
+        language_query: list,
+        task: str = "qa",
+        reset_ranking: bool = True
+) -> pd.DataFrame:
+    cols, _ = get_default_cols(task=task, columns=df.columns, add_fix_cols=False)
     selected_cols = []
     for c in cols:
-        if task == TaskType.qa:
-            eval_col = QABenchmarks[version_slug].value[c].value
-        elif task == TaskType.long_doc:
-            eval_col = LongDocBenchmarks[version_slug].value[c].value
-        else:
-            raise NotImplementedError
-        if eval_col.domain not in domains:
+        if task == "qa":
+            eval_col = BenchmarksQA[c].value
+        elif task == "long-doc":
+            eval_col = BenchmarksLongDoc[c].value
+        if eval_col.domain not in domain_query:
             continue
-        if eval_col.lang not in languages:
+        if eval_col.lang not in language_query:
             continue
         selected_cols.append(c)
     # We use COLS to maintain sorting
-    return selected_cols
-
-
-def select_columns(
-    df: pd.DataFrame,
-    domains: list,
-    languages: list,
-    task: TaskType = TaskType.qa,
-    reset_ranking: bool = True,
-    version_slug: str = None,
-) -> pd.DataFrame:
-    selected_cols = get_selected_cols(task, version_slug, domains, languages)
-    fixed_cols, _ = get_fixed_col_names_and_types()
-    filtered_df = df[fixed_cols + selected_cols]
-    filtered_df.replace({"": pd.NA}, inplace=True)
+    filtered_df = df[FIXED_COLS + selected_cols]
     if reset_ranking:
         filtered_df[COL_NAME_AVG] = filtered_df[selected_cols].apply(calculate_mean, axis=1).round(decimals=2)
         filtered_df.sort_values(by=[COL_NAME_AVG], ascending=False, inplace=True)
         filtered_df.reset_index(inplace=True, drop=True)
         filtered_df = reset_rank(filtered_df)
+
     return filtered_df
 
 
-def _update_df_elem(
-    task: TaskType,
-    version: str,
-    source_df: pd.DataFrame,
-    domains: list,
-    langs: list,
-    reranking_query: list,
-    query: str,
-    show_anonymous: bool,
-    reset_ranking: bool = True,
-    show_revision_and_timestamp: bool = False,
+def _update_table(
+        task: str,
+        hidden_df: pd.DataFrame,
+        domains: list,
+        langs: list,
+        reranking_query: list,
+        query: str,
+        show_anonymous: bool,
+        reset_ranking: bool = True,
+        show_revision_and_timestamp: bool = False
 ):
-    filtered_df = source_df.copy()
+    filtered_df = hidden_df.copy()
     if not show_anonymous:
         filtered_df = filtered_df[~filtered_df[COL_NAME_IS_ANONYMOUS]]
     filtered_df = filter_models(filtered_df, reranking_query)
     filtered_df = filter_queries(query, filtered_df)
-    filtered_df = select_columns(filtered_df, domains, langs, task, reset_ranking, get_safe_name(version))
+    filtered_df = select_columns(filtered_df, domains, langs, task, reset_ranking)
     if not show_revision_and_timestamp:
         filtered_df.drop([COL_NAME_REVISION, COL_NAME_TIMESTAMP], axis=1, inplace=True)
     return filtered_df
 
 
-def update_doc_df_elem(
-    version: str,
-    hidden_df: pd.DataFrame,
-    domains: list,
-    langs: list,
-    reranking_query: list,
-    query: str,
-    show_anonymous: bool,
-    show_revision_and_timestamp: bool = False,
-    reset_ranking: bool = True,
+def update_table(
+        hidden_df: pd.DataFrame,
+        domains: list,
+        langs: list,
+        reranking_query: list,
+        query: str,
+        show_anonymous: bool,
+        show_revision_and_timestamp: bool = False,
+        reset_ranking: bool = True
 ):
-    return _update_df_elem(
-        TaskType.long_doc,
-        version,
-        hidden_df,
-        domains,
-        langs,
-        reranking_query,
-        query,
-        show_anonymous,
-        reset_ranking,
-        show_revision_and_timestamp,
-    )
+    return _update_table(
+        "qa", hidden_df, domains, langs, reranking_query, query, show_anonymous, reset_ranking, show_revision_and_timestamp)
+
+
+def update_table_long_doc(
+        hidden_df: pd.DataFrame,
+        domains: list,
+        langs: list,
+        reranking_query: list,
+        query: str,
+        show_anonymous: bool,
+        show_revision_and_timestamp: bool = False,
+        reset_ranking: bool = True
+
+):
+    return _update_table(
+        "long-doc", hidden_df, domains, langs, reranking_query, query, show_anonymous, reset_ranking, show_revision_and_timestamp)
 
 
 def update_metric(
-    datastore,
-    task: TaskType,
-    metric: str,
-    domains: list,
-    langs: list,
-    reranking_model: list,
-    query: str,
-    show_anonymous: bool = False,
-    show_revision_and_timestamp: bool = False,
+        raw_data: List[FullEvalResult],
+        task: str,
+        metric: str,
+        domains: list,
+        langs: list,
+        reranking_model: list,
+        query: str,
+        show_anonymous: bool = False,
+        show_revision_and_timestamp: bool = False,
 ) -> pd.DataFrame:
-    if task == TaskType.qa:
-        update_func = update_qa_df_elem
-    elif task == TaskType.long_doc:
-        update_func = update_doc_df_elem
-    else:
-        raise NotImplementedError
-    df_elem = get_leaderboard_df(datastore, task=task, metric=metric)
-    version = datastore.version
-    return update_func(
-        version,
-        df_elem,
-        domains,
-        langs,
-        reranking_model,
-        query,
-        show_anonymous,
-        show_revision_and_timestamp,
-    )
+    if task == 'qa':
+        leaderboard_df = get_leaderboard_df(raw_data, task=task, metric=metric)
+        return update_table(
+            leaderboard_df,
+            domains,
+            langs,
+            reranking_model,
+            query,
+            show_anonymous,
+            show_revision_and_timestamp
+        )
+    elif task == "long-doc":
+        leaderboard_df = get_leaderboard_df(raw_data, task=task, metric=metric)
+        return update_table_long_doc(
+            leaderboard_df,
+            domains,
+            langs,
+            reranking_model,
+            query,
+            show_anonymous,
+            show_revision_and_timestamp
+        )
 
 
 def upload_file(filepath: str):
@@ -223,6 +218,7 @@ def upload_file(filepath: str):
     return filepath
 
 
+
 def get_iso_format_timestamp():
     # Get the current timestamp with UTC as the timezone
     current_timestamp = datetime.now(timezone.utc)
@@ -231,15 +227,15 @@ def get_iso_format_timestamp():
     current_timestamp = current_timestamp.replace(microsecond=0)
 
     # Convert to ISO 8601 format and replace the offset with 'Z'
-    iso_format_timestamp = current_timestamp.isoformat().replace("+00:00", "Z")
-    filename_friendly_timestamp = current_timestamp.strftime("%Y%m%d%H%M%S")
+    iso_format_timestamp = current_timestamp.isoformat().replace('+00:00', 'Z')
+    filename_friendly_timestamp = current_timestamp.strftime('%Y%m%d%H%M%S')
     return iso_format_timestamp, filename_friendly_timestamp
 
 
 def calculate_file_md5(file_path):
     md5 = hashlib.md5()
 
-    with open(file_path, "rb") as f:
+    with open(file_path, 'rb') as f:
         while True:
             data = f.read(4096)
             if not data:
@@ -250,14 +246,13 @@ def calculate_file_md5(file_path):
 
 
 def submit_results(
-    filepath: str,
-    model: str,
-    model_url: str,
-    reranking_model: str = "",
-    reranking_model_url: str = "",
-    version: str = LATEST_BENCHMARK_VERSION,
-    is_anonymous=False,
-):
+        filepath: str,
+        model: str,
+        model_url: str,
+        reranking_model: str="",
+        reranking_model_url: str="",
+        version: str="AIR-Bench_24.04",
+        is_anonymous=False):
     if not filepath.endswith(".zip"):
         return styled_error(f"file uploading aborted. wrong file type: {filepath}")
 
@@ -270,13 +265,11 @@ def submit_results(
         if not model_url.startswith("https://") and not model_url.startswith("http://"):
             # TODO: retrieve the model page and find the model name on the page
             return styled_error(
-                f"failed to submit. Model url must start with `https://` or `http://`. Illegal model url: {model_url}"
-            )
+                f"failed to submit. Model url must start with `https://` or `http://`. Illegal model url: {model_url}")
         if reranking_model != "NoReranker":
             if not reranking_model_url.startswith("https://") and not reranking_model_url.startswith("http://"):
                 return styled_error(
-                    f"failed to submit. Model url must start with `https://` or `http://`. Illegal model url: {model_url}"
-                )
+                    f"failed to submit. Model url must start with `https://` or `http://`. Illegal model url: {model_url}")
 
     # rename the uploaded file
     input_fp = Path(filepath)
@@ -286,15 +279,14 @@ def submit_results(
     input_folder_path = input_fp.parent
 
     if not reranking_model:
-        reranking_model = "NoReranker"
-
+        reranking_model = 'NoReranker'
+    
     API.upload_file(
         path_or_fileobj=filepath,
         path_in_repo=f"{version}/{model}/{reranking_model}/{output_fn}",
         repo_id=SEARCH_RESULTS_REPO,
         repo_type="dataset",
-        commit_message=f"feat: submit {model} to evaluate",
-    )
+        commit_message=f"feat: submit {model} to evaluate")
 
     output_config_fn = f"{output_fn.removesuffix('.zip')}.json"
     output_config = {
@@ -305,7 +297,7 @@ def submit_results(
         "version": f"{version}",
         "is_anonymous": is_anonymous,
         "revision": f"{revision}",
-        "timestamp": f"{timestamp_config}",
+        "timestamp": f"{timestamp_config}"
     }
     with open(input_folder_path / output_config_fn, "w") as f:
         json.dump(output_config, f, indent=4, ensure_ascii=False)
@@ -314,8 +306,7 @@ def submit_results(
         path_in_repo=f"{version}/{model}/{reranking_model}/{output_config_fn}",
         repo_id=SEARCH_RESULTS_REPO,
         repo_type="dataset",
-        commit_message=f"feat: submit {model} + {reranking_model} config",
-    )
+        commit_message=f"feat: submit {model} + {reranking_model} config")
     return styled_message(
         f"Thanks for submission!\n"
         f"Retrieval method: {model}\nReranking model: {reranking_model}\nSubmission revision: {revision}"
@@ -325,125 +316,3 @@ def submit_results(
 def reset_rank(df):
     df[COL_NAME_RANK] = df[COL_NAME_AVG].rank(ascending=False, method="min")
     return df
-
-
-def get_leaderboard_df(datastore, task: TaskType, metric: str) -> pd.DataFrame:
-    """
-    Creates a dataframe from all the individual experiment results
-    """
-    # load the selected metrics into a DataFrame from the raw json
-    all_data_json = []
-    for v in datastore.raw_data:
-        all_data_json += v.to_dict(task=task.value, metric=metric)
-    df = pd.DataFrame.from_records(all_data_json)
-
-    # calculate the average scores for selected task
-    if task == TaskType.qa:
-        benchmarks = QABenchmarks[datastore.slug]
-    elif task == TaskType.long_doc:
-        benchmarks = LongDocBenchmarks[datastore.slug]
-    else:
-        raise NotImplementedError
-    valid_cols = frozenset(df.columns.to_list())
-    benchmark_cols = []
-    for t in list(benchmarks.value):
-        if t.value.col_name not in valid_cols:
-            continue
-        benchmark_cols.append(t.value.col_name)
-
-    # filter out the columns that are not in the data
-    df[COL_NAME_AVG] = df[list(benchmark_cols)].apply(calculate_mean, axis=1).round(decimals=2)
-    df.sort_values(by=[COL_NAME_AVG], ascending=False, inplace=True)
-    df.reset_index(inplace=True, drop=True)
-
-    # filter out columns that are not in the data
-    display_cols = [COL_NAME_IS_ANONYMOUS, COL_NAME_AVG]
-    default_cols, _ = get_default_col_names_and_types(benchmarks)
-    for col in default_cols:
-        if col in valid_cols:
-            display_cols.append(col)
-    df = df[display_cols].round(decimals=2)
-
-    # rank the scores
-    df = reset_rank(df)
-
-    # shorten the revision
-    df[COL_NAME_REVISION] = df[COL_NAME_REVISION].str[:6]
-
-    return df
-
-
-def set_listeners(
-    task: TaskType,
-    target_df,
-    source_df,
-    search_bar,
-    version,
-    selected_domains,
-    selected_langs,
-    selected_rerankings,
-    show_anonymous,
-    show_revision_and_timestamp,
-):
-    if task == TaskType.qa:
-        update_table_func = update_qa_df_elem
-    elif task == TaskType.long_doc:
-        update_table_func = update_doc_df_elem
-    else:
-        raise NotImplementedError
-    selector_list = [selected_domains, selected_langs, selected_rerankings, search_bar, show_anonymous]
-    search_bar_args = [
-        source_df,
-        version,
-    ] + selector_list
-    selector_args = (
-        [version, source_df]
-        + selector_list
-        + [
-            show_revision_and_timestamp,
-        ]
-    )
-    # Set search_bar listener
-    search_bar.submit(update_table_func, search_bar_args, target_df)
-
-    # Set column-wise listener
-    for selector in selector_list:
-        selector.change(
-            update_table_func,
-            selector_args,
-            target_df,
-            queue=True,
-        )
-
-
-def update_qa_df_elem(
-    version: str,
-    hidden_df: pd.DataFrame,
-    domains: list,
-    langs: list,
-    reranking_query: list,
-    query: str,
-    show_anonymous: bool,
-    show_revision_and_timestamp: bool = False,
-    reset_ranking: bool = True,
-):
-    return _update_df_elem(
-        TaskType.qa,
-        version,
-        hidden_df,
-        domains,
-        langs,
-        reranking_query,
-        query,
-        show_anonymous,
-        reset_ranking,
-        show_revision_and_timestamp,
-    )
-
-
-def styled_error(error):
-    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
-
-
-def styled_message(message):
-    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"

tests/src/display/test_utils.py

@@ -0,0 +1,23 @@
+import pytest
+from src.display.utils import fields, AutoEvalColumnQA, COLS_QA, COLS_LONG_DOC, COLS_LITE, TYPES_QA, TYPES_LONG_DOC, QA_BENCHMARK_COLS, LONG_DOC_BENCHMARK_COLS, get_default_auto_eval_column_dict
+
+
+def test_fields():
+    for c in fields(AutoEvalColumnQA):
+        print(c)
+
+
+def test_macro_variables():
+    print(f'COLS_QA: {COLS_QA}')
+    print(f'COLS_LONG_DOC: {COLS_LONG_DOC}')
+    print(f'COLS_LITE: {COLS_LITE}')
+    print(f'TYPES_QA: {TYPES_QA}')
+    print(f'TYPES_LONG_DOC: {TYPES_LONG_DOC}')
+    print(f'QA_BENCHMARK_COLS: {QA_BENCHMARK_COLS}')
+    print(f'LONG_DOC_BENCHMARK_COLS: {LONG_DOC_BENCHMARK_COLS}')
+
+
+def test_get_default_auto_eval_column_dict():
+    auto_eval_column_dict_list = get_default_auto_eval_column_dict()
+    assert len(auto_eval_column_dict_list) == 9
+

tests/src/test_benchmarks.py

@@ -1,33 +1,9 @@
-import pytest
+from src.benchmarks import BenchmarksQA, BenchmarksLongDoc
 
-from src.benchmarks import LongDocBenchmarks, QABenchmarks
-from src.envs import BENCHMARK_VERSION_LIST
 
-# Ref: https://github.com/AIR-Bench/AIR-Bench/blob/4b27b8a8f2047a963805fcf6fb9d74be51ec440c/docs/available_tasks.md
-# 24.05
-# | Task | dev | test |
-# | ---- | --- | ---- |
-# | Long-Doc | 4 | 11 |
-# | QA | 54 | 53 |
-#
-# 24.04
-# | Task | test |
-# | ---- | ---- |
-# | Long-Doc | 15 |
-# | QA | 13 |
+def test_qabenchmarks():
+    print(list(BenchmarksQA))
 
 
-@pytest.mark.parametrize("num_datasets_dict", [{"air_bench_2404": 13, "air_bench_2405": 53}])
-def test_qa_benchmarks(num_datasets_dict):
-    assert len(QABenchmarks) == len(BENCHMARK_VERSION_LIST)
-    for benchmark_list in list(QABenchmarks):
-        version_slug = benchmark_list.name
-        assert num_datasets_dict[version_slug] == len(benchmark_list.value)
-
-
-@pytest.mark.parametrize("num_datasets_dict", [{"air_bench_2404": 15, "air_bench_2405": 11}])
-def test_doc_benchmarks(num_datasets_dict):
-    assert len(LongDocBenchmarks) == len(BENCHMARK_VERSION_LIST)
-    for benchmark_list in list(LongDocBenchmarks):
-        version_slug = benchmark_list.name
-        assert num_datasets_dict[version_slug] == len(benchmark_list.value)
+def test_longdocbenchmarks():
+    print(list(BenchmarksLongDoc))

tests/src/test_columns.py

@@ -1,119 +0,0 @@
-import pytest
-
-from src.benchmarks import LongDocBenchmarks, QABenchmarks
-from src.columns import (
-    COL_NAME_AVG,
-    COL_NAME_RANK,
-    COL_NAME_RERANKING_MODEL,
-    COL_NAME_RETRIEVAL_MODEL,
-    COL_NAME_REVISION,
-    COL_NAME_TIMESTAMP,
-    get_default_auto_eval_column_dict,
-    get_default_col_names_and_types,
-    get_fixed_col_names_and_types,
-    make_autoevalcolumn,
-)
-
-# Ref: https://github.com/AIR-Bench/AIR-Bench/blob/4b27b8a8f2047a963805fcf6fb9d74be51ec440c/docs/available_tasks.md
-# 24.05
-# | Task | dev | test |
-# | ---- | --- | ---- |
-# | Long-Doc | 4 | 11 |
-# | QA | 54 | 53 |
-#
-# 24.04
-# | Task | test |
-# | ---- | ---- |
-# | Long-Doc | 15 |
-# | QA | 13 |
-
-
-@pytest.fixture()
-def expected_col_names():
-    return [
-        "rank",
-        "retrieval_model",
-        "reranking_model",
-        "revision",
-        "timestamp",
-        "average",
-        "retrieval_model_link",
-        "reranking_model_link",
-        "is_anonymous",
-    ]
-
-
-@pytest.fixture()
-def expected_hidden_col_names():
-    return [
-        "retrieval_model_link",
-        "reranking_model_link",
-        "is_anonymous",
-    ]
-
-
-def test_get_default_auto_eval_column_dict(expected_col_names, expected_hidden_col_names):
-    col_list = get_default_auto_eval_column_dict()
-    assert len(col_list) == 9
-    hidden_cols = []
-    for col_tuple, expected_col in zip(col_list, expected_col_names):
-        col, _, col_content = col_tuple
-        assert col == expected_col
-        if col_content.hidden:
-            hidden_cols.append(col)
-    assert hidden_cols == expected_hidden_col_names
-
-
-def test_get_fixed_col_names_and_types():
-    col_names, col_types = get_fixed_col_names_and_types()
-    assert len(col_names) == 6
-    assert len(col_types) == 6
-    expected_col_and_type = [
-        (COL_NAME_RANK, "number"),
-        (COL_NAME_RETRIEVAL_MODEL, "markdown"),
-        (COL_NAME_RERANKING_MODEL, "markdown"),
-        (COL_NAME_REVISION, "markdown"),
-        (COL_NAME_TIMESTAMP, "date"),
-        (COL_NAME_AVG, "number"),
-    ]
-    for col_name, col_type, (c_name, c_type) in zip(col_names, col_types, expected_col_and_type):
-        assert col_name == c_name
-        assert col_type == c_type
-
-
-@pytest.mark.parametrize(
-    "benchmarks, expected_benchmark_len",
-    [
-        (QABenchmarks, {"air_bench_2404": 13, "air_bench_2405": 53}),
-        (LongDocBenchmarks, {"air_bench_2404": 15, "air_bench_2405": 11}),
-    ],
-)
-def test_make_autoevalcolumn(benchmarks, expected_benchmark_len, expected_col_names):
-    expected_default_attrs = frozenset(expected_col_names)
-    for benchmark in benchmarks:
-        TestEvalColumn = make_autoevalcolumn("TestEvalColumn", benchmark)
-        attrs = []
-        for k, v in TestEvalColumn.__dict__.items():
-            if not k.startswith("__"):
-                attrs.append(k)
-        attrs = frozenset(attrs)
-        assert expected_default_attrs.issubset(attrs)
-        benchmark_attrs = attrs.difference(expected_default_attrs)
-        assert len(benchmark_attrs) == expected_benchmark_len[benchmark.name]
-
-
-@pytest.mark.parametrize(
-    "benchmarks, expected_benchmark_len",
-    [
-        (QABenchmarks, {"air_bench_2404": 13, "air_bench_2405": 53}),
-        (LongDocBenchmarks, {"air_bench_2404": 15, "air_bench_2405": 11}),
-    ],
-)
-def test_get_default_col_names_and_types(
-    benchmarks, expected_benchmark_len, expected_col_names, expected_hidden_col_names
-):
-    default_col_len = len(expected_col_names)
-    hidden_col_len = len(expected_hidden_col_names)
-    for benchmark in benchmarks:
-        col_names, col_types = get_default_col_names_and_types(benchmark)
-        assert len(col_names) == expected_benchmark_len[benchmark.name] + default_col_len - hidden_col_len

tests/src/test_envs.py

@@ -1,14 +0,0 @@
-from air_benchmark.tasks import BenchmarkTable
-
-from src.envs import BENCHMARK_VERSION_LIST, DEFAULT_METRIC_LONG_DOC, DEFAULT_METRIC_QA, METRIC_LIST
-
-
-def test_benchmark_version_list():
-    leaderboard_versions = frozenset(BENCHMARK_VERSION_LIST)
-    available_versions = frozenset([k for k in BenchmarkTable.keys()])
-    assert leaderboard_versions.issubset(available_versions)
-
-
-def test_default_metrics():
-    assert DEFAULT_METRIC_QA in METRIC_LIST
-    assert DEFAULT_METRIC_LONG_DOC in METRIC_LIST

tests/src/test_loaders.py

@@ -1,46 +0,0 @@
-from pathlib import Path
-
-import pandas as pd
-import pytest
-
-from src.loaders import load_eval_results, load_leaderboard_datastore, load_raw_eval_results
-
-cur_fp = Path(__file__)
-
-
-@pytest.mark.parametrize("version", ["AIR-Bench_24.04", "AIR-Bench_24.05"])
-def test_load_raw_eval_results(version):
-    raw_data = load_raw_eval_results(cur_fp.parents[1] / f"toydata/eval_results/{version}")
-    assert len(raw_data) == 1
-    full_eval_result = raw_data[0]
-    expected_attr = [
-        "eval_name",
-        "retrieval_model",
-        "reranking_model",
-        "retrieval_model_link",
-        "reranking_model_link",
-        "results",
-        "timestamp",
-        "revision",
-        "is_anonymous",
-    ]
-    result_attr = [k for k in full_eval_result.__dict__.keys() if k[:2] != "__" and k[-2:] != "__"]
-    assert sorted(expected_attr) == sorted(result_attr)
-
-
-@pytest.mark.parametrize("version", ["AIR-Bench_24.04", "AIR-Bench_24.05"])
-def test_load_leaderboard_datastore(version):
-    file_path = cur_fp.parents[1] / f"toydata/eval_results/{version}"
-    datastore = load_leaderboard_datastore(file_path, version)
-    for k, v in datastore.__dict__.items():
-        if k[:2] != "__" and k[-2:] != "__":
-            if isinstance(v, list):
-                assert v
-            elif isinstance(v, pd.DataFrame):
-                assert not v.empty
-
-
-def test_load_eval_results():
-    file_path = cur_fp.parents[1] / "toydata/eval_results/"
-    datastore_dict = load_eval_results(file_path)
-    assert len(datastore_dict) == 2

tests/src/test_models.py

@@ -1,89 +0,0 @@
-from pathlib import Path
-
-import pytest
-
-from src.models import EvalResult, FullEvalResult
-
-cur_fp = Path(__file__)
-
-
-# Ref: https://github.com/AIR-Bench/AIR-Bench/blob/4b27b8a8f2047a963805fcf6fb9d74be51ec440c/docs/available_tasks.md
-# 24.05
-# | Task | dev | test |
-# | ---- | --- | ---- |
-# | Long-Doc | 4 | 11 |
-# | QA | 54 | 53 |
-#
-# 24.04
-# | Task | test |
-# | ---- | ---- |
-# | Long-Doc | 15 |
-# | QA | 13 |
-NUM_QA_BENCHMARKS_24_05 = 53
-NUM_DOC_BENCHMARKS_24_05 = 11
-NUM_QA_BENCHMARKS_24_04 = 13
-NUM_DOC_BENCHMARKS_24_04 = 15
-
-
-def test_eval_result():
-    EvalResult(
-        eval_name="eval_name",
-        retrieval_model="bge-m3",
-        reranking_model="NoReranking",
-        results=[{"domain": "law", "lang": "en", "dataset": "lex_files_500K-600K", "value": 0.45723}],
-        task="qa",
-        metric="ndcg_at_3",
-        timestamp="2024-05-14T03:09:08Z",
-        revision="1e243f14bd295ccdea7a118fe847399d",
-        is_anonymous=True,
-    )
-
-
-@pytest.mark.parametrize(
-    "file_path",
-    [
-        "AIR-Bench_24.04/bge-m3/jina-reranker-v2-base-multilingual/results.json",
-        "AIR-Bench_24.05/bge-m3/NoReranker/results.json",
-    ],
-)
-def test_full_eval_result_init_from_json_file(file_path):
-    json_fp = cur_fp.parents[1] / "toydata/eval_results/" / file_path
-    full_eval_result = FullEvalResult.init_from_json_file(json_fp)
-    assert json_fp.parents[0].stem == full_eval_result.reranking_model
-    assert json_fp.parents[1].stem == full_eval_result.retrieval_model
-    assert len(full_eval_result.results) == 70
-
-
-@pytest.mark.parametrize(
-    "file_path, task, expected_num_results",
-    [
-        ("AIR-Bench_24.04/bge-m3/jina-reranker-v2-base-multilingual/results.json", "qa", NUM_QA_BENCHMARKS_24_04),
-        (
-            "AIR-Bench_24.04/bge-m3/jina-reranker-v2-base-multilingual/results.json",
-            "long-doc",
-            NUM_DOC_BENCHMARKS_24_04,
-        ),
-        ("AIR-Bench_24.05/bge-m3/NoReranker/results.json", "qa", NUM_QA_BENCHMARKS_24_05),
-        ("AIR-Bench_24.05/bge-m3/NoReranker/results.json", "long-doc", NUM_DOC_BENCHMARKS_24_05),
-    ],
-)
-def test_full_eval_result_to_dict(file_path, task, expected_num_results):
-    json_fp = cur_fp.parents[1] / "toydata/eval_results/" / file_path
-    full_eval_result = FullEvalResult.init_from_json_file(json_fp)
-    result_dict_list = full_eval_result.to_dict(task)
-    assert len(result_dict_list) == 1
-    result = result_dict_list[0]
-    attr_list = frozenset(
-        [
-            "eval_name",
-            "Retrieval Method",
-            "Reranking Model",
-            "Retrieval Model LINK",
-            "Reranking Model LINK",
-            "Revision",
-            "Submission Date",
-            "Anonymous Submission",
-        ]
-    )
-    result_cols = list(result.keys())
-    assert len(result_cols) == (expected_num_results + len(attr_list))

tests/src/test_read_evals.py

@@ -0,0 +1,68 @@
+from pathlib import Path
+
+from src.read_evals import FullEvalResult, get_raw_eval_results, get_leaderboard_df
+
+cur_fp = Path(__file__)
+
+
+def test_init_from_json_file():
+    json_fp = cur_fp.parents[2] / "toydata" / "test_data.json"
+    full_eval_result = FullEvalResult.init_from_json_file(json_fp)
+    num_different_task_domain_lang_metric_dataset_combination = 6
+    assert len(full_eval_result.results) == \
+           num_different_task_domain_lang_metric_dataset_combination
+    assert full_eval_result.retrieval_model == "bge-m3"
+    assert full_eval_result.reranking_model == "bge-reranker-v2-m3"
+
+
+def test_to_dict():
+    json_fp = cur_fp.parents[2] / "toydata" / "test_data.json"
+    full_eval_result = FullEvalResult.init_from_json_file(json_fp)
+    result_list = full_eval_result.to_dict(task='qa', metric='ndcg_at_1')
+    assert len(result_list) == 1
+    result_dict = result_list[0]
+    assert result_dict["Retrieval Model"] == "bge-m3"
+    assert result_dict["Reranking Model"] == "bge-reranker-v2-m3"
+    assert result_dict["wiki_en"] is not None
+    assert result_dict["wiki_zh"] is not None
+
+
+def test_get_raw_eval_results():
+    results_path = cur_fp.parents[2] / "toydata" / "eval_results" / "AIR-Bench_24.04"
+    results = get_raw_eval_results(results_path)
+    # only load the latest results
+    assert len(results) == 4
+    assert results[0].eval_name == "bge-base-en-v1.5_NoReranker"
+    assert len(results[0].results) == 70
+    assert results[0].eval_name == "bge-base-en-v1.5_bge-reranker-v2-m3"
+    assert len(results[1].results) == 70
+
+
+def test_get_leaderboard_df():
+    results_path = cur_fp.parents[2] / "toydata" / "eval_results" / "AIR-Bench_24.04"
+    raw_data = get_raw_eval_results(results_path)
+    df = get_leaderboard_df(raw_data, 'qa', 'ndcg_at_10')
+    assert df.shape[0] == 4
+    # the results contain only one embedding model
+    # for i in range(4):
+    #     assert df["Retrieval Model"][i] == "bge-m3"
+    # # the results contain only two reranking model
+    # assert df["Reranking Model"][0] == "bge-reranker-v2-m3"
+    # assert df["Reranking Model"][1] == "NoReranker"
+    # assert df["Average ⬆️"][0] > df["Average ⬆️"][1]
+    # assert not df[['Average ⬆️', 'wiki_en', 'wiki_zh', ]].isnull().values.any()
+
+
+def test_get_leaderboard_df_long_doc():
+    results_path = cur_fp.parents[2] / "toydata" / "test_results"
+    raw_data = get_raw_eval_results(results_path)
+    df = get_leaderboard_df(raw_data, 'long-doc', 'ndcg_at_1')
+    assert df.shape[0] == 2
+    # the results contain only one embedding model
+    for i in range(2):
+        assert df["Retrieval Model"][i] == "bge-m3"
+    # the results contains only two reranking model
+    assert df["Reranking Model"][0] == "bge-reranker-v2-m3"
+    assert df["Reranking Model"][1] == "NoReranker"
+    assert df["Average ⬆️"][0] > df["Average ⬆️"][1]
+    assert not df[['Average ⬆️', 'law_en_lex_files_500k_600k', ]].isnull().values.any()

tests/src/test_utils.py

@@ -1,237 +0,0 @@
-from pathlib import Path
-
-import pandas as pd
-import pytest
-
-from src.columns import COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL
-from src.models import TaskType, model_hyperlink
-from src.utils import (
-    _update_df_elem,
-    calculate_mean,
-    filter_models,
-    filter_queries,
-    get_default_cols,
-    get_leaderboard_df,
-    get_selected_cols,
-    remove_html,
-    select_columns,
-)
-
-cur_fp = Path(__file__)
-
-NUM_QA_BENCHMARKS_24_05 = 53
-NUM_DOC_BENCHMARKS_24_05 = 11
-NUM_QA_BENCHMARKS_24_04 = 13
-NUM_DOC_BENCHMARKS_24_04 = 15
-
-
-@pytest.fixture
-def toy_df():
-    return pd.DataFrame(
-        {
-            "Retrieval Method": ["bge-m3", "bge-m3", "jina-embeddings-v2-base", "jina-embeddings-v2-base"],
-            "Reranking Model": ["bge-reranker-v2-m3", "NoReranker", "bge-reranker-v2-m3", "NoReranker"],
-            "Rank 🏆": [1, 2, 3, 4],
-            "Revision": ["123", "234", "345", "456"],
-            "Submission Date": ["", "", "", ""],
-            "Average ⬆️": [0.6, 0.4, 0.3, 0.2],
-            "wiki_en": [0.8, 0.7, 0.2, 0.1],
-            "wiki_zh": [0.4, 0.1, 0.4, 0.3],
-            "news_en": [0.8, 0.7, 0.2, 0.1],
-            "news_zh": [0.4, 0.1, 0.2, 0.3],
-            "Anonymous Submission": [False, False, False, True],
-        }
-    )
-
-
-def test_remove_html():
-    model_name = "jina-embeddings-v3"
-    html_str = model_hyperlink("https://jina.ai", model_name)
-    output_str = remove_html(html_str)
-    assert output_str == model_name
-
-
-def test_calculate_mean():
-    valid_row = [1, 3]
-    invalid_row = [2, pd.NA]
-    df = pd.DataFrame([valid_row, invalid_row], columns=["a", "b"])
-    result = list(df.apply(calculate_mean, axis=1))
-    assert result[0] == sum(valid_row) / 2
-    assert result[1] == -1
-
-
-@pytest.mark.parametrize(
-    "models, expected",
-    [
-        (["model1", "model3"], 2),
-        (["model1", "model_missing"], 1),
-        (["model1", "model2", "model3"], 3),
-        (
-            [
-                "model1",
-            ],
-            1,
-        ),
-        ([], 3),
-    ],
-)
-def test_filter_models(models, expected):
-    df = pd.DataFrame(
-        {
-            COL_NAME_RERANKING_MODEL: [
-                "model1",
-                "model2",
-                "model3",
-            ],
-            "col2": [1, 2, 3],
-        }
-    )
-    output_df = filter_models(df, models)
-    assert len(output_df) == expected
-
-
-@pytest.mark.parametrize(
-    "query, expected",
-    [
-        ("model1;model3", 2),
-        ("model1;model4", 1),
-        ("model1;model2;model3", 3),
-        ("model1", 1),
-        ("", 3),
-    ],
-)
-def test_filter_queries(query, expected):
-    df = pd.DataFrame(
-        {
-            COL_NAME_RETRIEVAL_MODEL: [
-                "model1",
-                "model2",
-                "model3",
-            ],
-            COL_NAME_RERANKING_MODEL: [
-                "model4",
-                "model5",
-                "model6",
-            ],
-        }
-    )
-    output_df = filter_queries(query, df)
-    assert len(output_df) == expected
-
-
-@pytest.mark.parametrize(
-    "task_type, slug, add_fix_cols, expected",
-    [
-        (TaskType.qa, "air_bench_2404", True, NUM_QA_BENCHMARKS_24_04),
-        (TaskType.long_doc, "air_bench_2404", True, NUM_DOC_BENCHMARKS_24_04),
-        (TaskType.qa, "air_bench_2405", False, NUM_QA_BENCHMARKS_24_05),
-        (TaskType.long_doc, "air_bench_2405", False, NUM_DOC_BENCHMARKS_24_05),
-    ],
-)
-def test_get_default_cols(task_type, slug, add_fix_cols, expected):
-    attr_cols = ["Rank 🏆", "Retrieval Method", "Reranking Model", "Revision", "Submission Date", "Average ⬆️"]
-    cols, types = get_default_cols(task_type, slug)
-    cols_set = frozenset(cols)
-    attrs_set = frozenset(attr_cols)
-    if add_fix_cols:
-        assert attrs_set.issubset(cols_set)
-    benchmark_cols = list(cols_set.difference(attrs_set))
-    assert len(benchmark_cols) == expected
-
-
-@pytest.mark.parametrize(
-    "task_type, domains, languages, expected",
-    [
-        (
-            TaskType.qa,
-            ["wiki", "news"],
-            [
-                "zh",
-            ],
-            ["wiki_zh", "news_zh"],
-        ),
-        (
-            TaskType.qa,
-            [
-                "law",
-            ],
-            ["zh", "en"],
-            ["law_en"],
-        ),
-        (
-            TaskType.long_doc,
-            ["healthcare"],
-            ["zh", "en"],
-            [
-                "healthcare_en_pubmed_100k_200k_1",
-                "healthcare_en_pubmed_100k_200k_2",
-                "healthcare_en_pubmed_100k_200k_3",
-                "healthcare_en_pubmed_40k_50k_5_merged",
-                "healthcare_en_pubmed_30k_40k_10_merged",
-            ],
-        ),
-    ],
-)
-def test_get_selected_cols(task_type, domains, languages, expected):
-    slug = "air_bench_2404"
-    cols = get_selected_cols(task_type, slug, domains, languages)
-    assert sorted(cols) == sorted(expected)
-
-
-@pytest.mark.parametrize("reset_rank", [False])
-def test_select_columns(toy_df, reset_rank):
-    expected = [
-        "Rank 🏆",
-        "Retrieval Method",
-        "Reranking Model",
-        "Revision",
-        "Submission Date",
-        "Average ⬆️",
-        "news_zh",
-    ]
-    df_result = select_columns(toy_df, ["news"], ["zh"], version_slug="air_bench_2404", reset_ranking=reset_rank)
-    assert len(df_result.columns) == len(expected)
-    if reset_rank:
-        assert df_result["Average ⬆️"].equals(df_result["news_zh"])
-    else:
-        assert df_result["Average ⬆️"].equals(toy_df["Average ⬆️"])
-
-
-@pytest.mark.parametrize(
-    "reset_rank, show_anony",
-    [
-        (False, True),
-        (True, True),
-        (True, False),
-    ],
-)
-def test__update_df_elem(toy_df, reset_rank, show_anony):
-    df = _update_df_elem(TaskType.qa, "AIR-Bench_24.04", toy_df, ["news"], ["zh"], [], "", show_anony, reset_rank)
-    if show_anony:
-        assert df.shape[0] == 4
-    else:
-        assert df.shape[0] == 3
-    if show_anony:
-        if reset_rank:
-            assert df["Average ⬆️"].equals(df["news_zh"])
-        else:
-            assert df["Average ⬆️"].equals(toy_df["Average ⬆️"])
-
-
-@pytest.mark.parametrize(
-    "version, task_type",
-    [
-        ("AIR-Bench_24.04", TaskType.qa),
-        ("AIR-Bench_24.04", TaskType.long_doc),
-        ("AIR-Bench_24.05", TaskType.qa),
-        ("AIR-Bench_24.05", TaskType.long_doc),
-    ],
-)
-def test_get_leaderboard_df(version, task_type):
-    from src.loaders import load_raw_eval_results
-    from src.models import LeaderboardDataStore, get_safe_name
-
-    raw_data = load_raw_eval_results(cur_fp.parents[1] / f"toydata/eval_results/{version}")
-    ds = LeaderboardDataStore(version, get_safe_name(version), raw_data=raw_data)
-    df = get_leaderboard_df(ds, task_type, "ndcg_at_10")
-    assert df.shape[0] == 1

tests/test_utils.py

@@ -0,0 +1,115 @@
+import pandas as pd
+import pytest
+
+from src.utils import filter_models, search_table, filter_queries, select_columns, update_table_long_doc, get_iso_format_timestamp, get_default_cols, update_table
+from src.display.utils import COL_NAME_IS_ANONYMOUS, COL_NAME_REVISION, COL_NAME_TIMESTAMP, COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL, COL_NAME_RANK, COL_NAME_AVG
+
+
+@pytest.fixture
+def toy_df():
+    return pd.DataFrame(
+        {
+            "Retrieval Model": [
+                "bge-m3",
+                "bge-m3",
+                "jina-embeddings-v2-base",
+                "jina-embeddings-v2-base"
+            ],
+            "Reranking Model": [
+                "bge-reranker-v2-m3",
+                "NoReranker",
+                "bge-reranker-v2-m3",
+                "NoReranker"
+            ],
+            "Average ⬆️": [0.6, 0.4, 0.3, 0.2],
+            "wiki_en": [0.8, 0.7, 0.2, 0.1],
+            "wiki_zh": [0.4, 0.1, 0.4, 0.3],
+            "news_en": [0.8, 0.7, 0.2, 0.1],
+            "news_zh": [0.4, 0.1, 0.4, 0.3],
+        }
+    )
+
+
+@pytest.fixture
+def toy_df_long_doc():
+    return pd.DataFrame(
+        {
+            "Retrieval Model": [
+                "bge-m3",
+                "bge-m3",
+                "jina-embeddings-v2-base",
+                "jina-embeddings-v2-base"
+            ],
+            "Reranking Model": [
+                "bge-reranker-v2-m3",
+                "NoReranker",
+                "bge-reranker-v2-m3",
+                "NoReranker"
+            ],
+            "Average ⬆️": [0.6, 0.4, 0.3, 0.2],
+            "law_en_lex_files_300k_400k": [0.4, 0.1, 0.4, 0.3],
+            "law_en_lex_files_400k_500k": [0.8, 0.7, 0.2, 0.1],
+            "law_en_lex_files_500k_600k": [0.8, 0.7, 0.2, 0.1],
+            "law_en_lex_files_600k_700k": [0.4, 0.1, 0.4, 0.3],
+        }
+    )
+def test_filter_models(toy_df):
+    df_result = filter_models(toy_df, ["bge-reranker-v2-m3", ])
+    assert len(df_result) == 2
+    assert df_result.iloc[0]["Reranking Model"] == "bge-reranker-v2-m3"
+
+
+def test_search_table(toy_df):
+    df_result = search_table(toy_df, "jina")
+    assert len(df_result) == 2
+    assert df_result.iloc[0]["Retrieval Model"] == "jina-embeddings-v2-base"
+
+
+def test_filter_queries(toy_df):
+    df_result = filter_queries("jina", toy_df)
+    assert len(df_result) == 2
+    assert df_result.iloc[0]["Retrieval Model"] == "jina-embeddings-v2-base"
+
+
+def test_select_columns(toy_df):
+    df_result = select_columns(toy_df, ['news',], ['zh',])
+    assert len(df_result.columns) == 4
+    assert df_result['Average ⬆️'].equals(df_result['news_zh'])
+
+
+def test_update_table_long_doc(toy_df_long_doc):
+    df_result = update_table_long_doc(toy_df_long_doc, ['law',], ['en',], ["bge-reranker-v2-m3", ], "jina")
+    print(df_result)
+
+
+def test_get_iso_format_timestamp():
+    timestamp_config, timestamp_fn = get_iso_format_timestamp()
+    assert len(timestamp_fn) == 14
+    assert len(timestamp_config) == 20
+    assert timestamp_config[-1] == "Z"
+
+
+def test_get_default_cols():
+    cols, types = get_default_cols("qa")
+    for c, t in zip(cols, types):
+        print(f"type({c}): {t}")
+    assert len(frozenset(cols)) == len(cols)
+
+
+def test_update_table():
+    df = pd.DataFrame(
+        {
+            COL_NAME_IS_ANONYMOUS: [False, False, False],
+            COL_NAME_REVISION: ["a1", "a2", "a3"],
+            COL_NAME_TIMESTAMP: ["2024-05-12T12:24:02Z"] * 3,
+            COL_NAME_RERANKING_MODEL: ["NoReranker"] * 3,
+            COL_NAME_RETRIEVAL_MODEL: ["Foo"] * 3,
+            COL_NAME_RANK: [1, 2, 3],
+            COL_NAME_AVG: [0.1, 0.2, 0.3],  # unsorted values
+            "wiki_en": [0.1, 0.2, 0.3]
+        }
+    )
+    results = update_table(df, "wiki", "en", ["NoReranker"], "", show_anonymous=False, reset_ranking=False, show_revision_and_timestamp=False)
+    # keep the RANK as the same regardless of the unsorted averages
+    assert results[COL_NAME_RANK].to_list() == [1, 2, 3]
+

tests/toydata/test_data.json

@@ -0,0 +1,98 @@
+[
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "long_doc",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "law",
+        "lang": "en",
+        "dataset": "lex_files_500K-600K",
+        "value": 0.75723
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "long_doc",
+      "metric": "ndcg_at_3"
+    },
+    "results": [
+      {
+        "domain": "law",
+        "lang": "en",
+        "dataset": "lex_files_500K-600K",
+        "value": 0.69909
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "qa",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "en",
+        "dataset": "unknown",
+        "value": 0.69083
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "qa",
+      "metric": "ndcg_at_3"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "en",
+        "dataset": "unknown",
+        "value": 0.73359
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "qa",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "zh",
+        "dataset": "unknown",
+        "value": 0.78358
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "qa",
+      "metric": "ndcg_at_3"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "zh",
+        "dataset": "unknown",
+        "value": 0.78358
+      }
+    ]
+  }
+]

tests/toydata/test_results/bge-m3/NoReranker/results_2023-11-21T18-10-08.json

@@ -0,0 +1,98 @@
+[
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "NoReranker",
+      "task": "long_doc",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "law",
+        "lang": "en",
+        "dataset": "lex_files_500K-600K",
+        "value": 0.45723
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "NoReranker",
+      "task": "long_doc",
+      "metric": "ndcg_at_3"
+    },
+    "results": [
+      {
+        "domain": "law",
+        "lang": "en",
+        "dataset": "lex_files_500K-600K",
+        "value": 0.49909
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "NoReranker",
+      "task": "qa",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "en",
+        "dataset": "unknown",
+        "value": 0.49083
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "NoReranker",
+      "task": "qa",
+      "metric": "ndcg_at_3"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "en",
+        "dataset": "unknown",
+        "value": 0.43359
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "NoReranker",
+      "task": "qa",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "zh",
+        "dataset": "unknown",
+        "value": 0.78358
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "NoReranker",
+      "task": "qa",
+      "metric": "ndcg_at_3"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "zh",
+        "dataset": "unknown",
+        "value": 0.78358
+      }
+    ]
+  }
+]

tests/toydata/test_results/bge-m3/bge-reranker-v2-m3/results_2023-11-21T18-10-08.json

@@ -0,0 +1,98 @@
+[
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "long_doc",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "law",
+        "lang": "en",
+        "dataset": "lex_files_500K-600K",
+        "value": 0.75723
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "long_doc",
+      "metric": "ndcg_at_3"
+    },
+    "results": [
+      {
+        "domain": "law",
+        "lang": "en",
+        "dataset": "lex_files_500K-600K",
+        "value": 0.69909
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "qa",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "en",
+        "dataset": "unknown",
+        "value": 0.69083
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "qa",
+      "metric": "ndcg_at_3"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "en",
+        "dataset": "unknown",
+        "value": 0.73359
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "qa",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "zh",
+        "dataset": "unknown",
+        "value": 0.78358
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "qa",
+      "metric": "ndcg_at_3"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "zh",
+        "dataset": "unknown",
+        "value": 0.78358
+      }
+    ]
+  }
+]