import gradio as gr import os from apscheduler.schedulers.background import BackgroundScheduler from huggingface_hub import HfApi from datetime import datetime, timedelta from src.assets.text_content import TITLE, INTRODUCTION_TEXT, CLEMSCORE_TEXT, MULTIMODAL_NAME, TEXT_NAME, HF_REPO from src.leaderboard_utils import query_search, get_github_data from src.plot_utils import split_models, plotly_plot, get_plot_df, update_open_models, update_closed_models from src.plot_utils import reset_show_all, reset_show_names, reset_show_legend, reset_mobile_view from src.version_utils import get_version_data from src.trend_utils import get_final_trend_plot """ CONSTANTS """ # For restarting the gradio application every 24 Hrs TIME = 43200 # in seconds # Reload will not work locally - requires HFToken # The app launches locally as expected - only without the reload utility """ AUTO RESTART HF SPACE """ HF_TOKEN = os.environ.get("H4_TOKEN", None) api = HfApi() def restart_space(): api.restart_space(repo_id=HF_REPO, token=HF_TOKEN) """ GITHUB UTILS """ github_data = get_github_data() text_leaderboard = github_data["text"]["dataframes"][0] # Get the latest version of text-only leaderboard multimodal_leaderboard = github_data["multimodal"]["dataframes"][0] # Get the latest version of multimodal leaderboard # Show only First 4 columns for the leaderboards # Should be Model Name, Clemscore, %Played, and Quality Score text_leaderboard = text_leaderboard.iloc[:, :4] multimodal_leaderboard = multimodal_leaderboard.iloc[:, :4] """ VERSIONS UTILS """ versions_data = get_version_data() latest_version = versions_data['versions'][0]['name'] last_updated_date = versions_data['versions'][0]['last_updated'][0] version_names = [v['name'] for v in versions_data['versions']] global version_df version_df = versions_data['dataframes'][0] def select_version_df(name): for i, v in enumerate(versions_data['versions']): if v['name'] == name: return versions_data['dataframes'][i] """ MAIN APPLICATION """ hf_app = gr.Blocks() with hf_app: gr.HTML(TITLE) gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") with gr.Tabs(elem_classes="tab-buttons") as tabs: """ ####################### FIRST TAB - TEXT-LEADERBOARD ####################### """ with gr.TabItem(TEXT_NAME, elem_id="llm-benchmark-tab-table", id=0): with gr.Row(): search_bar = gr.Textbox( placeholder=" πŸ” Search for models - separate multiple queries with `;` and press ENTER...", show_label=False, elem_id="search-bar", ) leaderboard_table = gr.Dataframe( value=text_leaderboard, elem_id="text-leaderboard-table", interactive=False, visible=True ) # Show information about the clemscore and last updated date below the table gr.HTML(CLEMSCORE_TEXT) gr.HTML(f"Last updated - {github_data['text']['version_data'][0]['last_updated'][0]}") # Add a dummy leaderboard to handle search queries in leaderboard_table # This will show a temporary leaderboard based on the searched value dummy_leaderboard_table = gr.Dataframe( value=text_leaderboard, elem_id="text-leaderboard-table-dummy", interactive=False, visible=False ) # Action after submitting a query to the search bar search_bar.submit( query_search, [dummy_leaderboard_table, search_bar], leaderboard_table, queue=True ) """ ####################### SECOND TAB - MULTIMODAL LEADERBOARD ####################### """ with gr.TabItem(MULTIMODAL_NAME, elem_id="mm-llm-benchmark-tab-table", id=1): with gr.Row(): mm_search_bar = gr.Textbox( placeholder=" πŸ” Search for models - separate multiple queries with `;` and press ENTER...", show_label=False, elem_id="search-bar", ) mm_leaderboard_table = gr.Dataframe( value=multimodal_leaderboard, elem_id="mm-leaderboard-table", interactive=False, visible=True ) # Show information about the clemscore and last updated date below the table gr.HTML(CLEMSCORE_TEXT) gr.HTML(f"Last updated - {github_data['multimodal']['version_data'][0]['last_updated'][0]}") # Add a dummy leaderboard to handle search queries in leaderboard_table # This will show a temporary leaderboard based on the searched value mm_dummy_leaderboard_table = gr.Dataframe( value=multimodal_leaderboard, elem_id="mm-leaderboard-table-dummy", interactive=False, visible=False ) # Action after submitting a query to the search bar mm_search_bar.submit( query_search, [mm_dummy_leaderboard_table, mm_search_bar], mm_leaderboard_table, queue=True ) """ ####################### THIRD TAB - PLOTS - %PLAYED V/S QUALITY SCORE ####################### """ with gr.TabItem("πŸ“Š Plots", elem_id="plots", id=2): """ DropDown Select for Text/Multimodal Leaderboard """ leaderboard_selection = gr.Dropdown( choices=[TEXT_NAME, MULTIMODAL_NAME], value=TEXT_NAME, label="Select Leaderboard πŸŽ–οΈπŸ”½", elem_id="value-select-0", interactive=True ) """ Accordion Groups to select individual models - Hidden by default """ with gr.Accordion("Select Open-weight Models 🌐", open=False): open_models_selection = update_open_models() clear_button_1 = gr.ClearButton(open_models_selection) with gr.Accordion("Select Commercial Models πŸ’°", open=False): closed_models_selection = update_closed_models() clear_button_2 = gr.ClearButton(closed_models_selection) """ Checkbox group to control the layout of the plot """ with gr.Row(): with gr.Column(): show_all = gr.CheckboxGroup( ["Select All Models"], label="Show plot for all models πŸ€–", value=[], elem_id="value-select-3", interactive=True, ) with gr.Column(): show_names = gr.CheckboxGroup( ["Show Names"], label="Show names of models on the plot 🏷️", value=[], elem_id="value-select-4", interactive=True, ) with gr.Column(): show_legend = gr.CheckboxGroup( ["Show Legend"], label="Show legend on the plot πŸ’‘", value=[], elem_id="value-select-5", interactive=True, ) with gr.Column(): mobile_view = gr.CheckboxGroup( ["Mobile View"], label="View plot on smaller screens πŸ“±", value=[], elem_id="value-select-6", interactive=True, ) """ PLOT BLOCK """ # Create a dummy DataFrame as an input to the plotly_plot function. # Uses this data to plot the %played v/s quality score with gr.Row(): dummy_plot_df = gr.DataFrame( value=get_plot_df(), visible=False ) with gr.Row(): with gr.Column(): # Output block for the plot plot_output = gr.Plot() """ PLOT CHANGE ACTIONS Toggle 'Select All Models' based on the values in Accordion checkbox groups """ open_models_selection.change( plotly_plot, [dummy_plot_df, open_models_selection, closed_models_selection, show_all, show_names, show_legend, mobile_view], [plot_output], queue=True ) closed_models_selection.change( plotly_plot, [dummy_plot_df, open_models_selection, closed_models_selection, show_all, show_names, show_legend, mobile_view], [plot_output], queue=True ) show_all.change( plotly_plot, [dummy_plot_df, open_models_selection, closed_models_selection, show_all, show_names, show_legend, mobile_view], [plot_output], queue=True ) show_names.change( plotly_plot, [dummy_plot_df, open_models_selection, closed_models_selection, show_all, show_names, show_legend, mobile_view], [plot_output], queue=True ) show_legend.change( plotly_plot, [dummy_plot_df, open_models_selection, closed_models_selection, show_all, show_names, show_legend, mobile_view], [plot_output], queue=True ) mobile_view.change( plotly_plot, [dummy_plot_df, open_models_selection, closed_models_selection, show_all, show_names, show_legend, mobile_view], [plot_output], queue=True ) """ LEADERBOARD SELECT CHANGE ACTIONS Update Checkbox Groups and Dummy DF based on the leaderboard selected """ leaderboard_selection.change( update_open_models, [leaderboard_selection], [open_models_selection], queue=True ) leaderboard_selection.change( update_closed_models, [leaderboard_selection], [closed_models_selection], queue=True ) leaderboard_selection.change( get_plot_df, [leaderboard_selection], [dummy_plot_df], queue=True ) ## Reset Plot when Leaderboard selection changes leaderboard_selection.change( reset_show_all, outputs=[show_all], queue=True ) open_models_selection.change( reset_show_all, outputs=[show_all], queue=True ) closed_models_selection.change( reset_show_all, outputs=[show_all], queue=True ) leaderboard_selection.change( reset_show_names, outputs=[show_names], queue=True ) leaderboard_selection.change( reset_show_legend, outputs=[show_legend], queue=True ) leaderboard_selection.change( reset_mobile_view, outputs=[mobile_view], queue=True ) """ ####################### FOURTH TAB - TRENDS ####################### """ with gr.TabItem("πŸ“ˆTrends", elem_id="trends-tab", id=3): with gr.Row(): mkd_text = gr.Markdown("### Commercial v/s Open-Weight models - clemscore over time. The size of the circles represents the scaled value of the parameters of the models. Larger circles indicate higher parameter values.") with gr.Row(): with gr.Column(scale=3): trend_select = gr.Dropdown( choices=["Text", "Multimodal"], value=None, label="Select Benchmark πŸ”", elem_id="value-select-7", interactive=True, ) with gr.Column(scale=1): mobile_view = gr.CheckboxGroup( choices=["Mobile View"], value=[], label="View plot on smaller screens πŸ“±", elem_id="value-select-8", interactive=True, ) with gr.Row(): trend_plot = gr.Plot(show_label=False) trend_select.change( get_final_trend_plot, [trend_select, mobile_view], [trend_plot], queue=True ) mobile_view.change( get_final_trend_plot, [trend_select, mobile_view], [trend_plot], queue=True ) """ ####################### FIFTH TAB - VERSIONS AND DETAILS ####################### """ with gr.TabItem("πŸ”„ Versions and Details", elem_id="versions-details-tab", id=4): with gr.Row(): version_select = gr.Dropdown( version_names, label="Select Version πŸ•ΉοΈ", value=latest_version ) with gr.Row(): search_bar_prev = gr.Textbox( placeholder=" πŸ” Search for models - separate multiple queries with `;` and press ENTER...", show_label=False, elem_id="search-bar-3", ) prev_table = gr.Dataframe( value=version_df, elem_id="version-leaderboard-table", interactive=False, visible=True ) dummy_prev_table = gr.Dataframe( value=version_df, elem_id="version-dummy-leaderboard-table", interactive=False, visible=False ) gr.HTML(CLEMSCORE_TEXT) gr.HTML(f"Last updated - {last_updated_date}") search_bar_prev.submit( query_search, [dummy_prev_table, search_bar_prev], prev_table, queue=True ) version_select.change( select_version_df, [version_select], prev_table, queue=True ) # Update Dummy Leaderboard, when changing versions version_select.change( select_version_df, [version_select], dummy_prev_table, queue=True ) hf_app.load() hf_app.queue() # Add scheduler to auto-restart the HF space at every TIME interval and update every component each time scheduler = BackgroundScheduler() scheduler.add_job(restart_space, 'interval', seconds=TIME) scheduler.start() # Log current start time and scheduled restart time print(datetime.now()) print(f"Scheduled restart at {datetime.now() + timedelta(seconds=TIME)}") hf_app.launch()