app.py

import os

import regex as re
import gradio as gr
import pandas as pd
from gradio.themes.utils.sizes import text_md
from gradio_modal import Modal

from content import (
    HEADER_MARKDOWN,
    LEADERBOARD_TAB_TITLE_MARKDOWN,
    SUBMISSION_TAB_TITLE_MARKDOWN,
    MODAL_SUBMIT_MARKDOWN,
    SUBMISSION_DETAILS_MARKDOWN,
    RANKING_AFTER_SUBMISSION_MARKDOWN,
    MORE_DETAILS_MARKDOWN,
)
from server import LeaderboardServer, xmlAndMarkdownEscape, xmlQuoteAttr

leaderboard_server = LeaderboardServer()


SUBMISSION_INPUTS = dict.fromkeys((
    "team_name",
    "model_name",
    "model_type",
    "parameters",
    "input_length",
    "precision",
    "description",
    "link_to_model",
    "submission_file",
)).keys()

def on_submit_pressed():
    return gr.update(value='Processing submission...', interactive=False)

def validate_submission_inputs(**inputs):
    if any(key for key, value in inputs.items() if key != "description" and value in (None, "")):
        raise ValueError('Please fill in all fields (only the description field is optional)')
    if not os.path.exists(inputs["submission_file"]):
        raise ValueError('File does not exist')
    if not (inputs["link_to_model"].startswith("http://") or inputs["link_to_model"].startswith("https://")):
        raise ValueError('Link does not starts with "http://" or "https://"')
    if not inputs["parameters"] > 0:
        raise ValueError('Attribute `Parameters (B)` should be greater than zero')
    if not (inputs["input_length"] > 0 and inputs["input_length"] == int(inputs["input_length"])):
        raise ValueError('Attribute `Input length (# tokens)` should be greater than zero and integer type')

def process_submission(*inputs):
    try:
        inputs = dict(zip(SUBMISSION_INPUTS, inputs))
        for key in inputs:
            if key in ("team_name", "model_name"):
                inputs[key] = re.sub(r"""\s+""", " ", inputs[key]).strip()
            elif key in ("description", "link_to_model"):
                inputs[key] = inputs[key].strip()
        validate_submission_inputs(**inputs)
        metadata = SUBMISSION_INPUTS - {"submission_file"}
        metadata = {key: inputs[key] for key in metadata}

        gr.Info('Submission valid, going to queue for the tournament...')

        pre_submit = leaderboard_server.prepare_model_for_submission(inputs["submission_file"], metadata)
    except ValueError as err:
        gr.Warning(str(err))
        return (
            gr.update(value='Pre-submit model', visible=True, interactive=True),
            gr.update(visible=False),
            gr.update(visible=False),
            gr.update(visible=False),
            gr.update(visible=False),
            gr.update(visible=False),
        )
    except Exception as err:
        gr.Warning(str(err), duration=None)
        return (
            gr.update(value='Pre-submit model', visible=True, interactive=True),
            gr.update(visible=False),
            gr.update(visible=False),
            gr.update(visible=False),
            gr.update(visible=False),
            gr.update(visible=False),
        )
    
    return (
        gr.update(visible=False),
        gr.update(visible=True),
        gr.update(interactive=True, visible=True),
        gr.update(interactive=True, visible=True),
        gr.update(visible=True),
        gr.update(
            value=leaderboard_server.get_leaderboard(pre_submit),
            visible=True,
            datatype="markdown",
            elem_classes="leaderboard-table",
        ),
    )

def get_submission_ids_and_titles():
    with leaderboard_server.var_lock.ro:
        submission_ids_and_titles = [
            (
                leaderboard_server.submission_id_to_model_title[submission_id],
                submission_id,
            )
            for submission_id in leaderboard_server.submission_ids
        ]
    
    submission_ids_and_titles.sort(key=lambda x: x[0].lower())
    
    return submission_ids_and_titles

def submit_results():
    leaderboard_server.save_pre_submit()
    leaderboard_server.update_leaderboard()
    gr.Info('Submission successful!')
    
    with leaderboard_server.var_lock.ro:
        leaderboard = leaderboard_server.get_leaderboard(category=leaderboard_server.TASKS_CATEGORY_OVERALL)
        submission_ids_and_titles = get_submission_ids_and_titles()
    
    return (
        gr.update(value='Pre-submit model', visible=True, interactive=True),
        gr.update(visible=False),
        gr.update(visible=False),
        gr.update(visible=False),
        gr.update(visible=False),
        gr.update(visible=False),
        gr.DataFrame(value=leaderboard, visible=True),
        gr.update(visible=False),
        gr.update(choices=submission_ids_and_titles),
        gr.update(value=leaderboard_server.TASKS_CATEGORY_OVERALL),
        gr.update(choices=submission_ids_and_titles),
    )


def erase_pre_submit():
    with leaderboard_server.pre_submit_lock:
        if leaderboard_server.pre_submit:
            leaderboard_server.pre_submit = None  # NOTE: Is it safe? How to confirm that `submission_id` is equal?
    return (
        gr.update(value='Pre-submit model', visible=True, interactive=True),
        gr.update(visible=False),
        gr.update(visible=False),
        gr.update(visible=False),
        gr.update(visible=False),
        gr.update(visible=False),
        gr.update(visible=False),
    )


def fetch_model_detail(submission_id):
    metadata = leaderboard_server.get_model_detail(submission_id)
    return (
        gr.update(value=metadata['description'], visible=True),
        gr.update(value=metadata['link_to_model'], visible=True)
    )

def fetch_model_tournament_results_table(submission_id, category):
    if submission_id == None:
        return gr.update(
            visible=False,
        )
    else:
        return gr.update(
            value=leaderboard_server.get_model_tournament_table(submission_id, category),
            visible=True,
        )

def create_task_abbreviation_legend_table(category):
    task_abbreviation_legend_body = []
    abbreviation2name = leaderboard_server.CATEGORY_TO_TASK_ABBREVIATION_TO_DETAILS[category]
    for abbr, name, url in abbreviation2name.values():
        task_abbreviation_legend_body.append([
            xmlAndMarkdownEscape(abbr),
            xmlAndMarkdownEscape(name),
            f'<a href={xmlQuoteAttr(url)}>{xmlAndMarkdownEscape(url)}</a>',
        ])
    
    return task_abbreviation_legend_body

def change_leaderboard_category(category, selected_submission_id):
    if category == leaderboard_server.TASKS_CATEGORY_OVERALL:
        task_abbreviation_legend = gr.update(
            visible=False,
        )
        tournament_results_title = gr.update(
            visible=False,
        )
        tournament_results_dropdown = gr.update(
            visible=False,
        )
        model_tournament_results_table = gr.update(
            visible=False,
        )
    else:
        task_abbreviation_legend = gr.update(
            value=create_task_abbreviation_legend_table(category),
            visible=True,
        )
        
        tournament_results_title = gr.update(
            visible=True,
        )
        
        tournament_results_dropdown = gr.update(
            visible=True,
        )
        
        model_tournament_results_table = fetch_model_tournament_results_table(selected_submission_id, category)
    
    return (
        gr.update(
            value=leaderboard_server.get_leaderboard(category=category),
            visible=True,
            datatype="markdown",
        ),
        task_abbreviation_legend,
        tournament_results_title,
        tournament_results_dropdown,
        model_tournament_results_table,
    )

def show_modal():
    return gr.update(visible=True)


def hide_modal():
    return gr.update(visible=False)


def on_application_load():
    leaderboard_server.update_leaderboard()
    
    with leaderboard_server.var_lock.ro:
        leaderboard = leaderboard_server.get_leaderboard(category=leaderboard_server.TASKS_CATEGORY_OVERALL)
        submission_ids_and_titles = get_submission_ids_and_titles()
    
    return (
        gr.update(
            value=leaderboard,
            visible=True,
        ),
        gr.update(choices=submission_ids_and_titles),
        gr.update(value=leaderboard_server.TASKS_CATEGORY_OVERALL),
        gr.update(choices=submission_ids_and_titles),
    )


custom_css = """

footer {visibility: hidden}

tr {
  background-color: var(--table-even-background-fill);
  font-family: "IBM Plex Mono";
}

tr.row_odd {
  background-color: var(--table-odd-background-fill);
}

.leaderboard-table td:first-child p, .leaderboard-table-model-details td:first-child p {
  margin: 0px;
}

.leaderboard-table th:nth-child(5), .leaderboard-table td:nth-child(5) {
  border-right-width: 2px;
  border-right-color: var(--border-color-primary);
}

.leaderboard-table td:nth-child(5) p {
  font-weight: bolder;
}

"""

with gr.Blocks(theme=gr.themes.Soft(text_size=text_md), css=custom_css) as main:
    gr.Markdown(HEADER_MARKDOWN)

    with gr.Tabs():
        with leaderboard_server.var_lock.ro:
            submission_ids_and_titles = get_submission_ids_and_titles()
        
        with gr.TabItem('Leaderboard'):
            with gr.Column():
                gr.Markdown(LEADERBOARD_TAB_TITLE_MARKDOWN)

                with gr.Row():
                    category_of_tasks = gr.Dropdown(
                        choices=[leaderboard_server.TASKS_CATEGORY_OVERALL] + sorted(leaderboard_server.TASKS_CATEGORIES),
                        value=leaderboard_server.TASKS_CATEGORY_OVERALL,
                        label="Category of benchmarks",
                        interactive=True,
                    )
                
                with gr.Row():
                    results_table = gr.DataFrame(
                        leaderboard_server.get_leaderboard(category=leaderboard_server.TASKS_CATEGORY_OVERALL),
                        interactive=False,
                        label=None,
                        visible=True,
                        datatype="markdown",
                        elem_classes="leaderboard-table",
                    )
                
                with gr.Row():
                    results_table_legend = gr.DataFrame(
                        value=None,
                        headers=[
                            "Abbr.", # "task abbreviation"
                            "Name",
                            "URL",
                        ],
                        column_widths=["150px"],
                        datatype="markdown",
                        label="Descriptions of the tasks",
                        visible=False,
                        interactive=False,
                        elem_classes="leaderboard-table-legend",
                    )
                
                with gr.Row():
                    tournament_results_title = gr.Markdown(
                        value="## Tournament results for selected model",
                        visible=False,
                    )
                
                with gr.Row():
                    tournament_results_dropdown = gr.Dropdown(
                        value=None,
                        choices=submission_ids_and_titles,
                        label="Select model",
                        visible=False,
                        interactive=True,
                    )
                
                with gr.Row():
                    model_tournament_results_table = gr.DataFrame(
                        value=None,
                        datatype="markdown",
                        label="The model won against…",
                        visible=False,
                        interactive=False,
                        elem_classes="leaderboard-table-model-details",
                    )
                
                category_of_tasks.change(
                    fn=change_leaderboard_category,
                    inputs=[
                        category_of_tasks,
                        tournament_results_dropdown,
                    ],
                    outputs=[
                        results_table,
                        results_table_legend,
                        tournament_results_title,
                        tournament_results_dropdown,
                        model_tournament_results_table,
                    ],
                )
                
                tournament_results_dropdown.change(
                    fn=fetch_model_tournament_results_table,
                    inputs=[
                        tournament_results_dropdown,
                        category_of_tasks,
                    ],
                    outputs=model_tournament_results_table,
                )

        with gr.TabItem('Model details'):
            gr.Markdown(MORE_DETAILS_MARKDOWN)
            
            detail_dropdown = gr.Dropdown(
                choices=submission_ids_and_titles,
                label="Select model",
                interactive=True,
            )

            with gr.Row():
                model_description = gr.Text(value='', label='Model description', visible=False, interactive=False)
                model_url = gr.Text(value='', label='Model url', visible=False, interactive=False)

            detail_dropdown.change(
                fn=fetch_model_detail,
                inputs=[detail_dropdown],
                outputs=[model_description, model_url],
            )

        with gr.TabItem('Submission'):
            with gr.Column():
                gr.Markdown(SUBMISSION_TAB_TITLE_MARKDOWN)
                
                submission_inputs = dict.fromkeys(SUBMISSION_INPUTS)
                
                with gr.Row():
                    submission_inputs["team_name"] = gr.Textbox(label='Team name', type='text')
                    submission_inputs["model_name"] = gr.Textbox(label='Model name', type='text')
                    submission_inputs["model_type"] = gr.Dropdown(
                        label="Model type",
                        choices=("chat", "pretrained", "ensemble"),
                    )
                    submission_inputs["parameters"] = gr.Number(
                        label='Parameters (B)',
                        value=0.01,
                        step=0.01,
                    )

                with gr.Row():
                    submission_inputs["input_length"] = gr.Number(
                        label='Input length (# tokens)',
                        value=0,
                        step=1,
                    )
                    submission_inputs["precision"] = gr.Dropdown(
                        label="Precision",
                        choices=("float32", "bfloat32", "float16", "bfloat16", "8bit", "4bit"),
                    )
                    submission_inputs["description"] = gr.Textbox(label='Description', type='text')
                    submission_inputs["link_to_model"] = gr.Textbox(label='Link to model', type='text')

                submission_inputs["submission_file"] = gr.File(label='Upload your results', type='filepath')
                
                pre_submission_btn = gr.Button(value='Pre-submit model', interactive=True)

                submit_prompt = gr.Markdown(
                    SUBMISSION_DETAILS_MARKDOWN,
                    visible=False
                )

                pre_submit_info = gr.Markdown(
                    RANKING_AFTER_SUBMISSION_MARKDOWN,
                    visible=False
                )

                pre_submit_table = gr.DataFrame(pd.DataFrame(), interactive=False, label=None, visible=False)

                submission_btn_yes = gr.Button(value='Submit model', interactive=False, visible=False)
                submission_btn_no = gr.Button(value='Reverse process', interactive=False, visible=False)

                with Modal(visible=False) as modal_submit:
                    gr.Markdown(MODAL_SUBMIT_MARKDOWN)
                    modal_submit_yes = gr.Button("Yes", interactive=True)
                    modal_submit_no = gr.Button("No", interactive=True)

                pre_submission_btn.click(
                    fn=on_submit_pressed,
                    outputs=[pre_submission_btn],
                ).then(  # TODO: Zjistit proč to neběží konkurentně.
                    fn=process_submission,
                    inputs=list(submission_inputs.values()),
                    outputs=[
                        pre_submission_btn,
                        submit_prompt,
                        submission_btn_yes,
                        submission_btn_no,
                        pre_submit_info,
                        pre_submit_table,
                    ],
                )

                submission_btn_yes.click(
                    fn=show_modal,
                    outputs=[modal_submit]
                )

                modal_submit_yes.click(
                    fn=submit_results,
                    outputs=[
                        pre_submission_btn,
                        submission_btn_yes,
                        submission_btn_no,
                        submit_prompt,
                        pre_submit_info,
                        pre_submit_table,
                        results_table,
                        modal_submit,
                        detail_dropdown,
                        category_of_tasks,
                        tournament_results_dropdown,
                    ],
                )

                modal_submit_no.click(
                    fn=hide_modal,
                    outputs=[modal_submit]
                )

                submission_btn_no.click(
                    fn=erase_pre_submit,
                    outputs=[
                        pre_submission_btn,
                        submission_btn_yes,
                        submission_btn_no,
                        submit_prompt,
                        pre_submit_info,
                        pre_submit_table,
                    ],
                )
    
    main.load(
        on_application_load,
        inputs=None,
        outputs=[
            results_table,
            detail_dropdown,
            category_of_tasks,
            tournament_results_dropdown,
        ]
    )

main.launch()