Fix missing scores

src/leaderboard/read_evals.py

@@ -42,6 +42,12 @@ class EvalResult:
         with open(json_filepath) as fp:
             data = json.load(fp)
 
+        if 'human_eval_solidity_pass@1' not in data['results']:
+            data['results']['human_eval_solidity_pass@1'] = {'score': 0}
+            
+        if 'human_eval_solidity_pass@3' not in data['results']:
+            data['results']['human_eval_solidity_pass@3'] = {'score': 0}
+
         org, model = get_org_and_model_names_from_filepath(json_filepath)
         config = data.get("config")
 

src/populate.py

@@ -28,59 +28,6 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     return df
 
 
-# def get_evaluation_requests_df(save_path: str, cols: list) -> list[pd.DataFrame]:
-#     """Creates the different dataframes for the evaluation requestss requestes"""
-#     entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
-#     all_evals = []
-
-#     for entry in entries:
-#         if ".json" in entry:
-#             file_path = os.path.join(save_path, entry)
-#             try:
-#                 with open(file_path, encoding='utf-8') as fp:
-#                     data = json.load(fp)
-#             except UnicodeDecodeError as e:
-#                 print(f"Unicode decoding error in {file_path}: {e}")
-#                 continue
-
-#             # data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-#             model_name = get_model_name_from_filepath(file_path)
-#             data[EvalQueueColumn.model.name] = make_clickable_model(model_name)
-
-#             data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-
-#             all_evals.append(data)
-
-#         elif ".md" not in entry:
-#             # this is a folder
-#             sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
-#             for sub_entry in sub_entries:
-#                 file_path = os.path.join(save_path, entry, sub_entry)
-#                 try:
-#                     with open(file_path, encoding='utf-8') as fp:
-#                         data = json.load(fp)
-#                 except json.JSONDecodeError:
-#                     print(f"Error reading {file_path}")
-#                     continue
-#                 except UnicodeDecodeError as e:
-#                     print(f"Unicode decoding error in {file_path}: {e}")
-#                     continue
-
-#                 # data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-#                 model_name = get_model_name_from_filepath(file_path)
-#                 data[EvalQueueColumn.model.name] = make_clickable_model(model_name)
-
-#                 data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-#                 all_evals.append(data)
-
-#     pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
-#     running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-#     finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
-#     df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
-#     df_running = pd.DataFrame.from_records(running_list, columns=cols)
-#     df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
-#     return df_finished[cols], df_running[cols], df_pending[cols]
-
 def get_evaluation_requests_df(save_path: str, cols: list) -> list[pd.DataFrame]:
     """Creates the different dataframes for the evaluation requestss requested."""
     all_evals = []
@@ -94,7 +41,6 @@ def get_evaluation_requests_df(save_path: str, cols: list) -> list[pd.DataFrame]
             return None
 
         model_name = get_model_name_from_filepath(file_path)
-        # data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
         data[EvalQueueColumn.model.name] = make_clickable_model(model_name)
         data[EvalQueueColumn.revision.name] = data.get("revision", "main")
         return data