Merge branch 'main' of https://huggingface.co/spaces/pminervini/hallucinations-leaderboard into main

app.py

@@ -56,21 +56,14 @@ def restart_space():
     API.restart_space(repo_id=REPO_ID, token=H4_TOKEN)
 
 
-def init_space(update_model_type_with_open_llm=True):
+def init_space():
     dataset_df = get_dataset_summary_table(file_path='blog/Hallucination-Leaderboard-Summary.csv')
 
     if socket.gethostname() not in {'neuromancer'}:
         # sync model_type with open-llm-leaderboard
         ui_snapshot_download(repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30)
         ui_snapshot_download(repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30)
-    # if EVAL_REQUESTS_PATH_OPEN_LLM == '' then we will not update model_type with open-llm-leaderbaord
-    if update_model_type_with_open_llm:
-        from src.envs import EVAL_REQUESTS_PATH_OPEN_LLM, QUEUE_REPO_OPEN_LLM
-        ui_snapshot_download(repo_id=QUEUE_REPO_OPEN_LLM, local_dir=EVAL_REQUESTS_PATH_OPEN_LLM, repo_type="dataset", tqdm_class=None, etag_timeout=30)
-    else:
-        EVAL_REQUESTS_PATH_OPEN_LLM = ""
-
-    raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, EVAL_REQUESTS_PATH_OPEN_LLM, COLS, BENCHMARK_COLS)
+    raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, "", COLS, BENCHMARK_COLS)
 
     finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
     return dataset_df, original_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df

cli/sync-open-llm-cli.py

@@ -0,0 +1,91 @@
+import os
+import json
+import glob
+
+from tqdm import tqdm
+from huggingface_hub import HfApi, snapshot_download
+from src.backend.manage_requests import EvalRequest
+from src.backend.envs import EVAL_REQUESTS_PATH_BACKEND_SYNC
+from src.envs import QUEUE_REPO, API
+from src.envs import EVAL_REQUESTS_PATH_OPEN_LLM, QUEUE_REPO_OPEN_LLM
+from src.utils import my_snapshot_download
+
+def my_set_eval_request(api, json_filepath, hf_repo, local_dir):
+    for i in range(10):
+        try:
+            set_eval_request(api=api, json_filepath=json_filepath, hf_repo=hf_repo, local_dir=local_dir)
+            return
+        except Exception:
+            time.sleep(60)
+    return
+
+
+def set_eval_request(api: HfApi, json_filepath: str, hf_repo: str, local_dir: str):
+    """Updates a given eval request with its new status on the hub (running, completed, failed, ...)"""
+
+    with open(json_filepath) as fp:
+        data = json.load(fp)
+
+    with open(json_filepath, "w") as f:
+        f.write(json.dumps(data))
+
+    api.upload_file(path_or_fileobj=json_filepath, path_in_repo=json_filepath.replace(local_dir, ""),
+                    repo_id=hf_repo, repo_type="dataset")
+
+
+def get_request_file_for_model(data, requests_path):
+    model_name = data["model"]
+    precision = data["precision"]
+    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED and RUNNING"""
+    request_files = os.path.join(
+        requests_path,
+        f"{model_name}_eval_request_*.json",
+    )
+    request_files = glob.glob(request_files)
+
+    # Select correct request file (precision)
+    request_file = ""
+    request_files = sorted(request_files, reverse=True)
+
+    for tmp_request_file in request_files:
+        with open(tmp_request_file, "r") as f:
+            req_content = json.load(f)
+            if req_content["precision"] == precision.split(".")[-1]:
+                request_file = tmp_request_file
+    return request_file
+
+def update_model_type(data, requests_path):
+    open_llm_request_file = get_request_file_for_model(data, requests_path)
+
+    try:
+        with open(open_llm_request_file, "r") as f:
+            open_llm_request = json.load(f)
+            data["model_type"] = open_llm_request["model_type"]
+            return True, data
+    except:
+        return False, data
+
+
+def read_and_write_json_files(directory, requests_path_open_llm):
+    # Walk through the directory
+    for subdir, dirs, files in tqdm(os.walk(directory), desc="updating model type according to open llm leaderboard"):
+        for file in files:
+            # Check if the file is a JSON file
+            if file.endswith('.json'):
+                file_path = os.path.join(subdir, file)
+                # Open and read the JSON file
+                with open(file_path, 'r') as json_file:
+                    data = json.load(json_file)
+                sucess, data = update_model_type(data, requests_path_open_llm)
+                if sucess:
+                    with open(file_path, 'w') as json_file:
+                        json.dump(data, json_file)
+                    my_set_eval_request(api=API, json_filepath=file_path, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND_SYNC)
+
+
+
+
+if __name__ == "__main__":
+    my_snapshot_download(repo_id=QUEUE_REPO_OPEN_LLM, revision="main", local_dir=EVAL_REQUESTS_PATH_OPEN_LLM, repo_type="dataset", max_workers=60)
+    my_snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND_SYNC, repo_type="dataset", max_workers=60)
+    read_and_write_json_files(EVAL_REQUESTS_PATH_BACKEND_SYNC, EVAL_REQUESTS_PATH_OPEN_LLM)

src/backend/envs.py

@@ -59,6 +59,7 @@ class Tasks(Enum):
 
 
 EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
+EVAL_REQUESTS_PATH_BACKEND_SYNC = os.path.join(CACHE_PATH, "eval-queue-bk-sync")
 EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
 
 DEVICE = "cuda" if torch.cuda.is_available() else 'cpu'

src/leaderboard/read_evals.py

@@ -128,18 +128,6 @@ class EvalResult:
         except Exception as e:
             print(f"Could not find request file for {self.org}/{self.model} -- path: {requests_path} -- {e}")
 
-    def update_model_type_with_open_llm_request_file(self, open_llm_requests_path):
-        """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model_open_llm(open_llm_requests_path, self.full_model, self.precision.value.name)
-
-        if request_file:
-            try:
-                with open(request_file, "r") as f:
-                    request = json.load(f)
-                self.model_type = ModelType.from_str(request.get("model_type", "Unknown"))
-            except Exception as e:
-                pass
-
     def is_complete(self) -> bool:
         for task in Tasks:
             if task.value.benchmark not in self.results:
@@ -216,10 +204,23 @@ def get_request_file_for_model_open_llm(requests_path, model_name, precision):
                 request_file = tmp_request_file
     return request_file
 
+def update_model_type_with_open_llm_request_file(result, open_llm_requests_path):
+    """Finds the relevant request file for the current model and updates info with it"""
+    request_file = get_request_file_for_model_open_llm(open_llm_requests_path, result.full_model, result.precision.value.name)
+
+    if request_file:
+        try:
+            with open(request_file, "r") as f:
+                request = json.load(f)
+            open_llm_model_type = request.get("model_type", "Unknown")
+            if open_llm_model_type != "Unknown":
+                result.model_type = ModelType.from_str(open_llm_model_type)
+        except Exception as e:
+            pass
+    return result
 
 def get_raw_eval_results(results_path: str,
                          requests_path: str,
-                         requests_path_open_llm: Optional[str] = None,
                          is_backend: bool = False) -> list[EvalResult]:
     """From the path of the results folder root, extract all needed info for results"""
     model_result_filepaths = []
@@ -243,8 +244,6 @@ def get_raw_eval_results(results_path: str,
         # Creation of result
         eval_result = EvalResult.init_from_json_file(model_result_filepath, is_backend=is_backend)
         eval_result.update_with_request_file(requests_path)
-        if requests_path_open_llm is not None:
-            eval_result.update_model_type_with_open_llm_request_file(requests_path_open_llm)
         # Store results of same eval together
         eval_name = eval_result.eval_name
         if eval_name in eval_results.keys():

src/populate.py

@@ -1,13 +1,13 @@
 import json
 import os
-
+from tqdm import tqdm
 import copy
 import pandas as pd
 
 from src.display.formatting import has_no_nan_values, make_clickable_model
 from src.display.utils import AutoEvalColumn, EvalQueueColumn
 from src.leaderboard.filter_models import filter_models
-from src.leaderboard.read_evals import get_raw_eval_results, EvalResult
+from src.leaderboard.read_evals import get_raw_eval_results, EvalResult, update_model_type_with_open_llm_request_file
 
 from src.backend.envs import Tasks as BackendTasks
 from src.display.utils import Tasks
@@ -21,6 +21,9 @@ def get_leaderboard_df(results_path: str,
                        is_backend: bool = False) -> tuple[list[EvalResult], pd.DataFrame]:
     # Returns a list of EvalResult
     raw_data: list[EvalResult] = get_raw_eval_results(results_path, requests_path, requests_path_open_llm)
+    if requests_path_open_llm != "":
+        for result_idx in tqdm(range(len(raw_data)), desc="updating model type with open llm leaderboard"):
+            raw_data[result_idx] = update_model_type_with_open_llm_request_file(raw_data[result_idx], requests_path_open_llm)
 
     all_data_json_ = [v.to_dict() for v in raw_data if v.is_complete()]