import gradio as gr from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns import pandas as pd from apscheduler.schedulers.background import BackgroundScheduler from huggingface_hub import snapshot_download from src.about import ( CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, EVALUATION_QUEUE_TEXT, INTRODUCTION_TEXT, ABOUT_TEXT, TITLE, Training_Dataset, Testing_Type ) from src.display.css_html_js import custom_css from src.display.utils import ( BENCHMARK_COLS, COLS, EVAL_COLS, EVAL_TYPES, AutoEvalColumn, ModelType, fields, Precision ) from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN from src.populate import get_evaluation_queue_df, get_leaderboard_df from src.submission.submit import add_new_eval def restart_space(): API.restart_space(repo_id=REPO_ID) ### Space initialisation try: snapshot_download( repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN ) except Exception: restart_space() try: snapshot_download( repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN ) except Exception: restart_space() LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS) ( finished_eval_queue_df, pending_eval_queue_df, ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS) def init_leaderboard(dataframe): if dataframe is None or dataframe.empty: raise ValueError("Leaderboard DataFrame is empty or None.") with gr.Tabs(elem_classes="leaderboard-tabs") as leaderboard_tabs: for testing_type in Testing_Type: with gr.TabItem("Average Scores" if testing_type.value == "avg" else testing_type.name, elem_id=f"{testing_type.value}_Leaderboard"): if testing_type.value == "avg": gr.Markdown("The scores presented in this tab are averaged scores across all datasets.") try: leaderboard = Leaderboard( value=dataframe[dataframe["Testing Type"] == testing_type.name], datatype=[c.type for c in fields(AutoEvalColumn)], select_columns=SelectColumns( default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default], cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden], label="Select Columns to Display:", ), search_columns=[AutoEvalColumn.model_name.name], hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden], filter_columns=[ ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"), ColumnFilter(AutoEvalColumn.training_dataset_type.name, type="checkboxgroup", label="Training Dataset"), ColumnFilter( AutoEvalColumn.model_parameters.name, type="slider", min=0, max=10000, default=["0", "100"], label="Select the number of parameters (M)", ), ], bool_checkboxgroup_label="Hide Models", interactive=False, ) except: gr.Markdown("There are no submissions for this testing type yet.") def init_submissions(): with gr.Column(): with gr.Row(): gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text") with gr.Column(): with gr.Accordion( f"✅ Finished Evaluations ({len(finished_eval_queue_df)})", open=False, ): with gr.Row(): finished_eval_table = gr.components.Dataframe( value=finished_eval_queue_df, headers=EVAL_COLS, datatype=EVAL_TYPES, row_count=5, ) with gr.Accordion( f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})", open=False, ): with gr.Row(): pending_eval_table = gr.components.Dataframe( value=pending_eval_queue_df, headers=EVAL_COLS, datatype=EVAL_TYPES, row_count=5, ) with gr.Row(): gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text") with gr.Row(): with gr.Column(): model_name_textbox = gr.Textbox(label="Model name") model_link_textbox = gr.Textbox(label="Link to Model") model_backbone_textbox = gr.Dropdown( choices=["Original"], label="Model Backbone", value="Original", allow_custom_value=True, ) model_parameter_number = gr.Number(label="Model Parameter Count (M)", precision=1, minimum=0) precision = gr.Dropdown( choices=[i.name for i in Precision], label="Precision", multiselect=False, value="float32", interactive=True, ) paper_name_textbox = gr.Textbox(label="Paper Name") paper_link_textbox = gr.Textbox(label="Link To Paper") with gr.Column(): training_dataset = gr.Dropdown( choices=[i.value for i in Training_Dataset if i.value != Training_Dataset.Other.value], label="Training Dataset", multiselect=False, value=Training_Dataset.XCL.value, interactive=True, allow_custom_value=True, ) testing_type = gr.Dropdown( choices=[i.name for i in Testing_Type], label="Tested on", multiselect=False, value=Testing_Type.AVG.name, interactive=True, ) cmap_value = gr.Number(label="cmAP Performance", precision=2, minimum=0.00, maximum=1.00, step=0.01) auroc_value = gr.Number(label="AUROC Performance", precision=2, minimum=0.00, maximum=1.00, step=0.01) t1acc_value = gr.Number(label="T1-Acc Performance", precision=2, minimum=0.00, maximum=1.00, step=0.01) submit_button = gr.Button("Submit Eval") submission_result = gr.Markdown() submit_button.click( fn=add_new_eval, inputs=[ model_name_textbox, model_link_textbox, model_backbone_textbox, precision, model_parameter_number, paper_name_textbox, paper_link_textbox, training_dataset, testing_type, cmap_value, auroc_value, t1acc_value, ], outputs=submission_result, ) demo = gr.Blocks(css=custom_css) with demo: gr.HTML(TITLE) gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") with gr.Tabs(elem_classes="tab-buttons") as tabs: with gr.TabItem("🏅 Leaderboard", elem_id="leaderboard-tab-table", id=0): init_leaderboard(LEADERBOARD_DF) with gr.TabItem("📝 About", elem_id="leaderboard-tab-table", id=2): gr.Markdown(ABOUT_TEXT, elem_classes="markdown-text") with gr.TabItem("🚀 Submit here! ", elem_id="leaderboard-tab-table", id=3): init_submissions() with gr.Row(): with gr.Accordion("📙 Citation", open=False): citation_button = gr.Textbox( value=CITATION_BUTTON_TEXT, label=CITATION_BUTTON_LABEL, lines=20, elem_id="citation-button", show_copy_button=True, ) scheduler = BackgroundScheduler() scheduler.add_job(restart_space, "interval", seconds=1800) scheduler.start() demo.launch()