feat: adapt to the latest data format

app.py

@@ -27,12 +27,12 @@ try:
 except Exception:
     restart_space()
 
-raw_data = get_raw_eval_results(EVAL_RESULTS_PATH)
+raw_data = get_raw_eval_results(f"{EVAL_RESULTS_PATH}/AIR-Bench_24.04")
 
 original_df_qa = get_leaderboard_df(
     raw_data, task='qa', metric='ndcg_at_3')
 original_df_long_doc = get_leaderboard_df(
-    raw_data, task='long_doc', metric='ndcg_at_3')
+    raw_data, task='long-doc', metric='ndcg_at_3')
 print(f'raw data: {len(raw_data)}')
 print(f'QA data loaded: {original_df_qa.shape}')
 print(f'Long-Doc data loaded: {len(original_df_long_doc)}')
@@ -42,7 +42,7 @@ shown_columns_qa = get_default_cols('qa', leaderboard_df_qa.columns, add_fix_col
 leaderboard_df_qa = leaderboard_df_qa[shown_columns_qa]
 
 leaderboard_df_long_doc = original_df_long_doc.copy()
-shown_columns_long_doc = get_default_cols('long_doc', leaderboard_df_long_doc.columns, add_fix_cols=True)
+shown_columns_long_doc = get_default_cols('long-doc', leaderboard_df_long_doc.columns, add_fix_cols=True)
 leaderboard_df_long_doc = leaderboard_df_long_doc[shown_columns_long_doc]
 
 
@@ -62,7 +62,7 @@ def update_metric_long_doc(
         reranking_model: list,
         query: str,
 ):
-    return update_metric(raw_data, 'long_doc', metric, domains, langs, reranking_model, query)
+    return update_metric(raw_data, "long-doc", metric, domains, langs, reranking_model, query)
 
 
 demo = gr.Blocks(css=custom_css)

src/benchmarks.py

@@ -40,7 +40,7 @@ dataset_dict = {
         "arxiv": {
             "en": ["Arxiv", ]},
     },
-    "long_doc": {
+    "long-doc": {
         "arxiv": {
             "en": ["gpt-3", "llama2", "llm-survey", "gemini"],
         },
@@ -125,7 +125,7 @@ for task, domain_dict in dataset_dict.items():
                 col_name = benchmark_name
                 for metric in dataset_list:
                     qa_benchmark_dict[benchmark_name] = Benchmark(benchmark_name, metric, col_name, domain, lang, task)
-            elif task == "long_doc":
+            elif task == "long-doc":
                 for dataset in dataset_list:
                     benchmark_name = f"{domain}_{lang}_{dataset}"
                     benchmark_name = get_safe_name(benchmark_name)

src/display/utils.py

@@ -22,6 +22,8 @@ class ColumnContent:
 COL_NAME_AVG = "Average ⬆️"
 COL_NAME_RETRIEVAL_MODEL = "Retrieval Model"
 COL_NAME_RERANKING_MODEL = "Reranking Model"
+COL_NAME_RETRIEVAL_MODEL_LINK = "Retrieval Model LINK"
+COL_NAME_RERANKING_MODEL_LINK = "Reranking Model LINK"
 COL_NAME_RANK = "Rank 🏆"
 
 def make_autoevalcolumn(cls_name="BenchmarksQA", benchmarks=BenchmarksQA):
@@ -34,6 +36,12 @@ def make_autoevalcolumn(cls_name="BenchmarksQA", benchmarks=BenchmarksQA):
     auto_eval_column_dict.append(
         ["reranking_model", ColumnContent, ColumnContent(COL_NAME_RERANKING_MODEL, "markdown", True, never_hidden=True)]
     )
+    auto_eval_column_dict.append(
+        ["retrieval_model_link", ColumnContent, ColumnContent(COL_NAME_RETRIEVAL_MODEL, "markdown", False, hidden=True, never_hidden=False)]
+    )
+    auto_eval_column_dict.append(
+        ["reranking_model_link", ColumnContent, ColumnContent(COL_NAME_RERANKING_MODEL, "markdown", False, hidden=True, never_hidden=False)]
+    )
     auto_eval_column_dict.append(
         ["average", ColumnContent, ColumnContent(COL_NAME_AVG, "number", True)]
     )

src/leaderboard/read_evals.py

@@ -12,6 +12,8 @@ from src.display.formatting import has_no_nan_values
 from src.display.utils import (
     COL_NAME_RERANKING_MODEL,
     COL_NAME_RETRIEVAL_MODEL,
+    COL_NAME_RERANKING_MODEL_LINK,
+    COL_NAME_RETRIEVAL_MODEL_LINK,
     COLS_QA,
     QA_BENCHMARK_COLS,
     COLS_LONG_DOC,
@@ -44,6 +46,8 @@ class FullEvalResult:
     eval_name: str  # name of the evaluation, [retrieval_model]_[reranking_model]
     retrieval_model: str
     reranking_model: str
+    retrieval_model_link: str
+    reranking_model_link: str
     results: List[EvalResult]  # results on all the EvalResults over different tasks and metrics.
     date: str = ""
 
@@ -58,10 +62,15 @@ class FullEvalResult:
 
         # store all the results for different metrics and tasks
         result_list = []
+        retrieval_model_link = ""
+        reranking_model_link = ""
         for item in model_data:
             config = item.get("config", {})
             # eval results for different metrics
             results = item.get("results", [])
+            retrieval_model_link=config["retreival_model_link"]
+            if config["reranking_model_link"] is not None:
+                reranking_model_link=""
             eval_result = EvalResult(
                 eval_name=f"{config['retrieval_model']}_{config['reranking_model']}_{config['metric']}",
                 retrieval_model=config["retrieval_model"],
@@ -75,6 +84,8 @@ class FullEvalResult:
             eval_name=f"{result_list[0].retrieval_model}_{result_list[0].reranking_model}",
             retrieval_model=result_list[0].retrieval_model,
             reranking_model=result_list[0].reranking_model,
+            retrieval_model_link=retrieval_model_link,
+            reranking_model_link=reranking_model_link,
             results=result_list
         )
 
@@ -91,6 +102,8 @@ class FullEvalResult:
             results[eval_result.eval_name]["eval_name"] = eval_result.eval_name
             results[eval_result.eval_name][COL_NAME_RETRIEVAL_MODEL] = self.retrieval_model
             results[eval_result.eval_name][COL_NAME_RERANKING_MODEL] = self.reranking_model
+            results[eval_result.eval_name][COL_NAME_RETRIEVAL_MODEL_LINK] = self.retrieval_model_link
+            results[eval_result.eval_name][COL_NAME_RERANKING_MODEL_LINK] = self.reranking_model_link
 
             # print(f'result loaded: {eval_result.eval_name}')
             for result in eval_result.results:
@@ -99,9 +112,9 @@ class FullEvalResult:
                 lang = result["lang"]
                 dataset = result["dataset"]
                 value = result["value"]
-                if task == 'qa':
+                if dataset == 'default':
                     benchmark_name = f"{domain}_{lang}"
-                elif task == 'long_doc':
+                else:
                     benchmark_name = f"{domain}_{lang}_{dataset}"
                 results[eval_result.eval_name][get_safe_name(benchmark_name)] = value
         return [v for v in results.values()]
@@ -115,13 +128,12 @@ def get_raw_eval_results(results_path: str) -> List[FullEvalResult]:
     for root, dirs, files in os.walk(results_path):
         if len(files) == 0:
             continue
-        try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7], reverse=True)
-        except dateutil.parser._parser.ParserError:
-            files = [files[-1]]
 
         # select the latest results
         for file in files:
+            if file != "results.json":
+                print(f'skip {file}')
+                continue
             model_result_filepaths.append(os.path.join(root, file))
 
     eval_results = {}
@@ -154,7 +166,7 @@ def get_leaderboard_df(raw_data: List[FullEvalResult], task: str, metric: str) -
     if task == "qa":
         cols = COLS_QA
         benchmark_cols = QA_BENCHMARK_COLS
-    elif task == "long_doc":
+    elif task == "long-doc":
         cols = COLS_LONG_DOC
         benchmark_cols = LONG_DOC_BENCHMARK_COLS
     else:

tests/src/leaderboard/test_read_evals.py

@@ -28,35 +28,35 @@ def test_to_dict():
 
 
 def test_get_raw_eval_results():
-    results_path = cur_fp.parents[2] / "toydata" / "test_results" / "bge-m3"
+    results_path = cur_fp.parents[2] / "toydata" / "eval_results" / "AIR-Bench_24.04"
     results = get_raw_eval_results(results_path)
     # only load the latest results
-    assert len(results) == 2
-    assert results[0].eval_name == "bge-m3_NoReranker"
-    assert len(results[0].results) == 6
-    assert results[1].eval_name == "bge-m3_bge-reranker-v2-m3"
-    assert len(results[1].results) == 6
+    assert len(results) == 4
+    assert results[0].eval_name == "bge-base-en-v1.5_NoReranker"
+    assert len(results[0].results) == 70
+    assert results[0].eval_name == "bge-base-en-v1.5_bge-reranker-v2-m3"
+    assert len(results[1].results) == 70
 
 
 def test_get_leaderboard_df():
-    results_path = cur_fp.parents[2] / "toydata" / "test_results"
+    results_path = cur_fp.parents[2] / "toydata" / "eval_results" / "AIR-Bench_24.04"
     raw_data = get_raw_eval_results(results_path)
-    df = get_leaderboard_df(raw_data, 'qa', 'ndcg_at_1')
-    assert df.shape[0] == 2
+    df = get_leaderboard_df(raw_data, 'qa', 'ndcg_at_3')
+    assert df.shape[0] == 4
     # the results contain only one embedding model
-    for i in range(2):
-        assert df["Retrieval Model"][i] == "bge-m3"
-    # the results contain only two reranking model
-    assert df["Reranking Model"][0] == "bge-reranker-v2-m3"
-    assert df["Reranking Model"][1] == "NoReranker"
-    assert df["Average ⬆️"][0] > df["Average ⬆️"][1]
-    assert not df[['Average ⬆️', 'wiki_en', 'wiki_zh', ]].isnull().values.any()
+    # for i in range(4):
+    #     assert df["Retrieval Model"][i] == "bge-m3"
+    # # the results contain only two reranking model
+    # assert df["Reranking Model"][0] == "bge-reranker-v2-m3"
+    # assert df["Reranking Model"][1] == "NoReranker"
+    # assert df["Average ⬆️"][0] > df["Average ⬆️"][1]
+    # assert not df[['Average ⬆️', 'wiki_en', 'wiki_zh', ]].isnull().values.any()
 
 
 def test_get_leaderboard_df_long_doc():
     results_path = cur_fp.parents[2] / "toydata" / "test_results"
     raw_data = get_raw_eval_results(results_path)
-    df = get_leaderboard_df(raw_data, 'long_doc', 'ndcg_at_1')
+    df = get_leaderboard_df(raw_data, 'long-doc', 'ndcg_at_1')
     assert df.shape[0] == 2
     # the results contain only one embedding model
     for i in range(2):

utils.py

@@ -47,7 +47,7 @@ def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
 def get_default_cols(task: str, columns: list, add_fix_cols: bool=True) -> list:
     if task == "qa":
         cols = list(frozenset(COLS_QA).intersection(frozenset(BENCHMARK_COLS_QA)).intersection(frozenset(columns)))
-    elif task == "long_doc":
+    elif task == "long-doc":
         cols = list(frozenset(COLS_LONG_DOC).intersection(frozenset(BENCHMARK_COLS_LONG_DOC)).intersection(frozenset(columns)))
     else:
         raise NotImplemented
@@ -68,7 +68,7 @@ def select_columns(df: pd.DataFrame, domain_query: list, language_query: list, t
     for c in cols:
         if task == "qa":
             eval_col = BenchmarksQA[c].value
-        elif task == "long_doc":
+        elif task == "long-doc":
             eval_col = BenchmarksLongDoc[c].value
         if eval_col.domain not in domain_query:
             continue
@@ -127,7 +127,7 @@ def update_metric(
             reranking_model,
             query
         )
-    elif task == 'long_doc':
+    elif task == "long-doc":
         leaderboard_df = get_leaderboard_df(raw_data, task=task, metric=metric)
         return update_table_long_doc(
             leaderboard_df,