display button unification for benchmarks

app.py

@@ -2,10 +2,11 @@
 import os
 import datetime
 import socket
+from threading import Thread
 
 import gradio as gr
 import pandas as pd
-
+import time
 from apscheduler.schedulers.background import BackgroundScheduler
 
 from huggingface_hub import snapshot_download
@@ -35,13 +36,27 @@ from src.display.utils import (
     fields,
     WeightType,
     Precision,
+    GPUType
 )
 
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, H4_TOKEN, IS_PUBLIC, QUEUE_REPO, REPO_ID, RESULTS_REPO
+from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, H4_TOKEN, IS_PUBLIC, \
+    QUEUE_REPO, REPO_ID, RESULTS_REPO, DEBUG_QUEUE_REPO, DEBUG_RESULTS_REPO
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
 from src.utils import get_dataset_summary_table
 
+def get_args():
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Run the LLM Leaderboard")
+    parser.add_argument("--debug", action="store_true", help="Run in debug mode")
+    return parser.parse_args()
+
+args = get_args()
+if args.debug:
+    print("Running in debug mode")
+    QUEUE_REPO = DEBUG_QUEUE_REPO
+    RESULTS_REPO = DEBUG_RESULTS_REPO
 
 def ui_snapshot_download(repo_id, local_dir, repo_type, tqdm_class, etag_timeout):
     try:
@@ -75,17 +90,13 @@ def init_space():
     )
     return dataset_df, original_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df
 
-
-dataset_df, original_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = init_space()
-leaderboard_df = original_df.copy()
-
-
 # Searching and filtering
 def update_table(
     hidden_df: pd.DataFrame, columns: list, type_query: list, precision_query: list, size_query: list, query: str
 ):
     filtered_df = filter_models(hidden_df, type_query, size_query, precision_query)
     filtered_df = filter_queries(query, filtered_df)
+    columns.extend(add_benchmark_columns(columns))
     df = select_columns(filtered_df, columns)
     return df
 
@@ -141,7 +152,62 @@ def filter_models(df: pd.DataFrame, type_query: list, size_query: list, precisio
     # filtered_df = filtered_df.loc[mask]
 
     return filtered_df
+    
+    
+def add_benchmark_columns(shown_columns):
+    benchmark_columns = []
+    for benchmark in BENCHMARK_COLS:
+        if benchmark in shown_columns:
+            for c in COLS:
+                if benchmark in c and benchmark != c:
+                    benchmark_columns.append(c)
+    return benchmark_columns
+
+shown_columns = None
+dataset_df, original_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = init_space()
+leaderboard_df = original_df.copy()
+
+# def update_leaderboard_table():
+#     global leaderboard_df, shown_columns
+#     print("Updating leaderboard table")
+#     return leaderboard_df[
+#                 [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
+#                 + shown_columns.value
+#                 + [AutoEvalColumn.dummy.name]
+#             ] if not leaderboard_df.empty else leaderboard_df
+        
+
+# def update_hidden_leaderboard_table():
+#     global original_df
+#     return original_df[COLS] if original_df.empty is False else original_df
+
+# def update_dataset_table():
+#     global dataset_df
+#     return dataset_df
 
+# def update_finish_table():
+#     global finished_eval_queue_df
+#     return finished_eval_queue_df
+
+# def update_running_table():
+#     global running_eval_queue_df
+#     return running_eval_queue_df
+
+# def update_pending_table():
+#     global pending_eval_queue_df
+#     return pending_eval_queue_df
+
+# def update_finish_num():
+#     global finished_eval_queue_df
+#     return len(finished_eval_queue_df)
+
+# def update_running_num():
+#     global running_eval_queue_df
+#     return len(running_eval_queue_df)
+
+# def update_pending_num():
+#     global pending_eval_queue_df
+#     return len(pending_eval_queue_df)
 
 # triggered only once at startup => read query parameter if it exists
 def load_query(request: gr.Request):
@@ -162,7 +228,7 @@ with demo:
                         search_bar = gr.Textbox(
                             placeholder=" 🔍 Model search (separate multiple queries with `;`)",
                             show_label=False,
-                            elem_id="search-bar",
+                            elem_id="search-bar"
                         )
                     with gr.Row():
                         shown_columns = gr.CheckboxGroup(
@@ -215,18 +281,19 @@ with demo:
                     # )
 
             # breakpoint()
-
+            benchmark_columns = add_benchmark_columns(shown_columns.value)       
             leaderboard_table = gr.components.Dataframe(
                 value=(
                     leaderboard_df[
                         [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
                         + shown_columns.value
+                        + benchmark_columns
                         + [AutoEvalColumn.dummy.name]
                     ]
                     if leaderboard_df.empty is False
                     else leaderboard_df
                 ),
-                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value + benchmark_columns,
                 datatype=TYPES,
                 elem_id="leaderboard-table",
                 interactive=False,
@@ -251,14 +318,14 @@ with demo:
                     filter_columns_size,
                     search_bar,
                 ],
-                leaderboard_table,
+                leaderboard_table
             )
 
             # Check query parameter once at startup and update search bar
             demo.load(load_query, inputs=[], outputs=[search_bar])
 
             for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size]:
-                selector.change(
+                selector.select(
                     update_table,
                     [
                         hidden_leaderboard_table_for_search,
@@ -323,6 +390,15 @@ with demo:
                     value=None,
                     interactive=True,
                 )
+                
+                gpu_type = gr.Dropdown(
+                    choices=[t.to_str() for t in GPUType],
+                    label="GPU type",
+                    multiselect=False,
+                    value="NVIDIA-A100-PCIe-80GB",
+                    interactive=True,
+                )
+                
 
             with gr.Row():
                 with gr.Column():
@@ -358,6 +434,7 @@ with demo:
 
             submit_button = gr.Button("Submit Eval")
             submission_result = gr.Markdown()
+            debug = gr.Checkbox(value=args.debug, label="Debug", visible=False)
             submit_button.click(
                 add_new_eval,
                 [
@@ -369,6 +446,8 @@ with demo:
                     weight_type,
                     model_type,
                     inference_framework,
+                    debug,
+                    gpu_type
                 ],
                 submission_result,
             )
@@ -385,8 +464,7 @@ with demo:
 
 scheduler = BackgroundScheduler()
 
-scheduler.add_job(restart_space, "interval", seconds=6 * 60 * 60)
-
+scheduler.add_job(restart_space, "interval", hours=6)
 
 def launch_backend():
     import subprocess
@@ -395,8 +473,9 @@ def launch_backend():
     if DEVICE not in {"cpu"}:
         _ = subprocess.run(["python", "backend-cli.py"])
 
-
+# Thread(target=periodic_init, daemon=True).start()
 # scheduler.add_job(launch_backend, "interval", seconds=120)
-
-scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+if __name__ == "__main__":
+    scheduler.start()
+    demo.queue(default_concurrency_limit=40).launch()
+    

backend-cli.py

@@ -16,13 +16,13 @@ from src.backend.envs import Tasks, EVAL_REQUESTS_PATH_BACKEND, EVAL_RESULTS_PAT
 from src.backend.manage_requests import EvalRequest
 from src.leaderboard.read_evals import EvalResult
 
-from src.envs import QUEUE_REPO, RESULTS_REPO, API
+from src.envs import QUEUE_REPO, RESULTS_REPO, API, DEBUG_QUEUE_REPO, DEBUG_RESULTS_REPO
 from src.utils import my_snapshot_download, analyze_gpu_stats, parse_nvidia_smi, monitor_gpus
 
 from src.leaderboard.read_evals import get_raw_eval_results
 
 from typing import Optional
-
+import GPUtil
 import time
 
 import pprint
@@ -126,6 +126,9 @@ def request_to_result_name(request: EvalRequest) -> str:
 def process_evaluation(task: Task, eval_request: EvalRequest, limit: Optional[int] = None) -> dict:
     batch_size = 1
     batch_size = eval_request.batch_size
+    
+    if args.debug:
+        RESULTS_REPO = DEBUG_RESULTS_REPO
 
     init_gpu_info = analyze_gpu_stats(parse_nvidia_smi())
     # if init_gpu_info['Mem(M)'] > 500:
@@ -364,9 +367,22 @@ def maybe_refresh_results(thr: int, hard_task_lst: Optional[list[str]] = None) -
     return False
 
 
+def get_gpu_details():
+    gpus = GPUtil.getGPUs()
+    gpu = gpus[0]
+    name = gpu.name.replace(" ", "-")
+    # Convert memory from MB to GB and round to nearest whole number
+    memory_gb = round(gpu.memoryTotal / 1024)
+    memory = f"{memory_gb}GB"
+    formatted_name = f"{name}-{memory}"
+    return formatted_name
+
 def process_pending_requests() -> bool:
+    if args.debug:
+        QUEUE_REPO = DEBUG_QUEUE_REPO
+        
     sanity_checks()
-
+    print("Processing pending requests")
     current_pending_status = [PENDING_STATUS]
 
     # Get all eval request that are PENDING, if you want to run other evals, change this parameter
@@ -385,6 +401,12 @@ def process_pending_requests() -> bool:
 
     eval_request = eval_requests[0]
     pp.pprint(eval_request)
+    
+    gpu_type = eval_request.gpu_type
+    curr_gpu_type = get_gpu_details()
+    if gpu_type != curr_gpu_type:
+        print(f"GPU type mismatch: {gpu_type} vs {curr_gpu_type}")
+        return False
 
     my_snapshot_download(
         repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60
@@ -426,6 +448,8 @@ def get_args():
     parser.add_argument("--precision", type=str, default="float32,float16,8bit,4bit", help="Precision to debug")
     parser.add_argument("--inference-framework", type=str, default="hf-chat", help="Inference framework to debug")
     parser.add_argument("--limit", type=int, default=None, help="Limit for the number of samples")
+    parser.add_argument("--gpu-type", type=str, default="NVIDIA-A100-PCIe-80GB", 
+                        help="GPU type. NVIDIA-A100-PCIe-80GB; NVIDIA-RTX-A5000-24GB; NVIDIA-H100-PCIe-80GB")
     return parser.parse_args()
 
 
@@ -454,8 +478,13 @@ if __name__ == "__main__":
                             status="",
                             json_filepath="",
                             precision=precision,  # Use precision from arguments
-                            inference_framework=args.inference_framework  # Use inference framework from arguments
+                            inference_framework=args.inference_framework,  # Use inference framework from arguments
+                            gpu_type=args.gpu_type
                         )
+                        curr_gpu_type = get_gpu_details()
+                        if eval_request.gpu_type != curr_gpu_type:
+                            print(f"GPU type mismatch: {eval_request.gpu_type} vs {curr_gpu_type}")
+                            raise Exception("GPU type mismatch")
                         results = process_evaluation(task, eval_request, limit=args.limit)
                     except Exception as e:
                         print(f"debug running error: {e}")

requirements.txt

@@ -27,6 +27,7 @@ cchardet
 rouge_score
 bert-score
 evaluate
-spacy
+spacy==3.7.4
 selfcheckgpt
 immutabledict
+gputil

src/backend/manage_requests.py

@@ -28,6 +28,7 @@ class EvalRequest:
     params: Optional[int] = None
     license: Optional[str] = ""
     batch_size: Optional[int] = 1
+    gpu_type: Optional[str] = "NVIDIA-A100-PCIe-80GB"
 
     def get_model_args(self) -> str:
         model_args = f"pretrained={self.model},revision={self.revision},parallelize=True"  # ,max_length=4096"

src/display/utils.py

@@ -104,16 +104,16 @@ auto_eval_column_dict.append(["inference_framework", ColumnContent, ColumnConten
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
     # System performance metrics
-    auto_eval_column_dict.append([f"{task.name}_end_to_end_time", ColumnContent, ColumnContent(f"{task.value.col_name} {E2Es}", "number", True)])
-    auto_eval_column_dict.append([f"{task.name}_batch_size", ColumnContent, ColumnContent(f"{task.value.col_name} {BATCH_SIZE}", "number", True)])
-    # auto_eval_column_dict.append([f"{task.name}_precision", ColumnContent, ColumnContent(f"{task.value.col_name} {PRECISION}", "str", True)])
-    auto_eval_column_dict.append([f"{task.name}_gpu_mem", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_Mem}", "number", True)])
-    auto_eval_column_dict.append([f"{task.name}_gpu", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_Name}", "str", True)])
-    auto_eval_column_dict.append([f"{task.name}_gpu_util", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_Util}", "number", True)])
+    auto_eval_column_dict.append([f"{task.name}_end_to_end_time", ColumnContent, ColumnContent(f"{task.value.col_name} {E2Es}", "number", True, hidden=True)])
+    auto_eval_column_dict.append([f"{task.name}_batch_size", ColumnContent, ColumnContent(f"{task.value.col_name} {BATCH_SIZE}", "number", True, hidden=True)])
+    # auto_eval_column_dict.append([f"{task.name}_precision", ColumnContent, ColumnContent(f"{task.value.col_name} {PRECISION}", "str", True, hidden=True)])
+    auto_eval_column_dict.append([f"{task.name}_gpu_mem", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_Mem}", "number", True, hidden=True)])
+    auto_eval_column_dict.append([f"{task.name}_gpu", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_Name}", "str", True, hidden=True)])
+    auto_eval_column_dict.append([f"{task.name}_gpu_util", ColumnContent, ColumnContent(f"{task.value.col_name} {GPU_Util}", "number", True, hidden=True)])
     if task.value.benchmark in MULTIPLE_CHOICEs:
         continue
-    # auto_eval_column_dict.append([f"{task.name}_prefilling_time", ColumnContent, ColumnContent(f"{task.value.col_name} {PREs}", "number", False)])
-    auto_eval_column_dict.append([f"{task.name}_decoding_throughput", ColumnContent, ColumnContent(f"{task.value.col_name} {TS}", "number", True)])
+    # auto_eval_column_dict.append([f"{task.name}_prefilling_time", ColumnContent, ColumnContent(f"{task.value.col_name} {PREs}", "number", False, hidden=True)])
+    auto_eval_column_dict.append([f"{task.name}_decoding_throughput", ColumnContent, ColumnContent(f"{task.value.col_name} {TS}", "number", True, hidden=True)])
 
 
 # Model information
@@ -140,6 +140,7 @@ class EvalQueueColumn:  # Queue column
     private = ColumnContent("private", "bool", True)
     precision = ColumnContent("precision", "str", True)
     weight_type = ColumnContent("weight_type", "str", "Original")
+    model_framework = ColumnContent("inference_framework", "str", True)
     status = ColumnContent("status", "str", True)
 
 
@@ -189,7 +190,25 @@ class InferenceFramework(Enum):
             return InferenceFramework.HF_Chat
         return InferenceFramework.Unknown
 
+class GPUType(Enum):
+    H100_pcie = ModelDetails("NVIDIA-H100-PCIe-80GB")
+    A100_pcie = ModelDetails("NVIDIA-A100-PCIe-80GB")
+    A5000 = ModelDetails("NVIDIA-RTX-A5000-24GB")
+    Unknown = ModelDetails("?")
 
+    def to_str(self):
+        return self.value.name
+
+    @staticmethod
+    def from_str(gpu_type: str):
+        if gpu_type in ["NVIDIA-H100-PCIe-80GB"]:
+            return GPUType.A100_pcie
+        if gpu_type in ["NVIDIA-A100-PCIe-80GB"]:
+            return GPUType.H100_pcie
+        if gpu_type in ["NVIDIA-A5000-24GB"]:
+            return GPUType.A5000
+        return GPUType.Unknown
+    
 class WeightType(Enum):
     Adapter = ModelDetails("Adapter")
     Original = ModelDetails("Original")
@@ -223,8 +242,8 @@ class Precision(Enum):
 
 
 # Column selection
-COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
-TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
+COLS = [c.name for c in fields(AutoEvalColumn)]
+TYPES = [c.type for c in fields(AutoEvalColumn)]
 COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
 TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
 

src/envs.py

@@ -12,8 +12,8 @@ QUEUE_REPO = "sparse-generative-ai/requests"
 QUEUE_REPO_OPEN_LLM = "open-llm-leaderboard/requests"
 RESULTS_REPO = "sparse-generative-ai/results"
 
-PRIVATE_QUEUE_REPO = "sparse-generative-ai/private-requests"
-PRIVATE_RESULTS_REPO = "sparse-generative-ai/private-results"
+DEBUG_QUEUE_REPO = "sparse-generative-ai/debug_requests"
+DEBUG_RESULTS_REPO = "sparse-generative-ai/debug_results"
 
 IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))
 

src/populate.py

@@ -95,6 +95,7 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> tuple[pd.DataFrame, p
 
             data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
             data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+            data[EvalQueueColumn.model_framework.name] = data.get("inference_framework", "-")
 
             all_evals.append(data)
         elif ".md" not in entry:
@@ -107,6 +108,7 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> tuple[pd.DataFrame, p
 
                 data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
                 data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+                data[EvalQueueColumn.model_framework.name] = data.get("inference_framework", "-")
                 all_evals.append(data)
 
     pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]

src/submission/check_validity.py

@@ -130,7 +130,7 @@ def already_submitted_models(requested_models_dir: str) -> set[str]:
                     continue
                 with open(os.path.join(root, file), "r") as f:
                     info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}_{info['inference_framework']}")
+                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}_{info['inference_framework']}_{info['gpu_type']}")
 
                     # Select organisation
                     if info["model"].count("/") == 0 or "submitted_time" not in info:

src/submission/submit.py

@@ -3,7 +3,7 @@ import os
 from datetime import datetime, timezone
 
 from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, H4_TOKEN, QUEUE_REPO, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA
+from src.envs import API, EVAL_REQUESTS_PATH, H4_TOKEN, QUEUE_REPO, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA, DEBUG_QUEUE_REPO
 from src.leaderboard.filter_models import DO_NOT_SUBMIT_MODELS
 from src.submission.check_validity import (
     already_submitted_models,
@@ -26,12 +26,17 @@ def add_new_eval(
     weight_type: str,
     model_type: str,
     inference_framework: str,
+    debug: bool = False,
+    gpu_type: str = "NVIDIA-A100-PCIe-80GB",
 ):
     global REQUESTED_MODELS
     global USERS_TO_SUBMISSION_DATES
     if not REQUESTED_MODELS:
         REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
 
+    if debug:
+        QUEUE_REPO = DEBUG_QUEUE_REPO
+    
     user_name = ""
     model_path = model
     if "/" in model:
@@ -110,17 +115,18 @@ def add_new_eval(
         "params": model_size,
         "license": license,
         "inference_framework": inference_framework,
+        "gpu_type": gpu_type
     }
 
     # Check for duplicate submission
-    if f"{model}_{revision}_{precision}_{inference_framework}" in REQUESTED_MODELS:
+    if f"{model}_{revision}_{precision}_{inference_framework}_{gpu_type}" in REQUESTED_MODELS:
         return styled_warning("This model has been already submitted.")
 
     print("Creating eval file")
     OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
     os.makedirs(OUT_DIR, exist_ok=True)
     # out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{precision}_{weight_type}.json"
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{precision}_{weight_type}_{inference_framework}.json"
+    out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{precision}_{weight_type}_{inference_framework}_{gpu_type}.json"
 
     with open(out_path, "w") as f:
         f.write(json.dumps(eval_entry))