"""A gradio app that renders a static leaderboard. This is used for Hugging Face Space.""" import argparse import gradio as gr import numpy as np import pandas as pd import gradio as gr import pandas as pd import json from constants import BANNER, INTRODUCTION_TEXT, CITATION_TEXT, METRICS_TAB_TEXT, DIR_OUTPUT_REQUESTS LAST_UPDATED = "Feb 28th 2024" css = """ .markdown-text{font-size: 15pt} .markdown-text-small{font-size: 13pt} th { text-align: center; } td { font-size: 15px; /* Adjust the font size as needed */ text-align: center; } #od-benchmark-tab-table-button{ font-size: 15pt; font-weight: bold; } """ column_names = { "model": "Model", "Overall": "All 🎯", "Turn 1": "Turn 1️⃣", "Turn 2": "Turn 2️⃣", } model_info = { "gpt-4": {"hf_name": "https://platform.openai.com/", "pretty_name": "gpt-4"}, "gpt-3.5-turbo": {"hf_name": "https://platform.openai.com/", "pretty_name": "gpt-3.5-turbo"}, "Llama-2-70b-hf": {"hf_name": "meta-llama/Llama-2-70b-hf", "pretty_name": "Llama-2-70B"}, "Llama-2-13b-hf": {"hf_name": "meta-llama/Llama-2-13b-hf", "pretty_name": "Llama-2-13B"}, "Llama-2-7b-hf": {"hf_name": "meta-llama/Llama-2-7b-hf", "pretty_name": "Llama-2-7B"}, "Mixtral-8x7B-v0.1": {"hf_name": "mistralai/Mixtral-8x7B-v0.1", "pretty_name": "Mixtral-8x7B"}, "Mistral-7b-v0.1": {"hf_name": "mistralai/Mistral-7B-v0.1", "pretty_name": "Mistral-7B v0.1"}, "Mistral-7b-v0.2": {"hf_name": "alpindale/Mistral-7B-v0.2-hf", "pretty_name": "Mistral-7B v0.2"}, "Yi-34B": {"hf_name": "01-ai/Yi-34B", "pretty_name": "Yi-34B"}, "Yi-6B": {"hf_name": "01-ai/Yi-6B", "pretty_name": "Yi-6B"}, "gemma-7b": {"hf_name": "google/gemma-7b", "pretty_name": "Gemma-7B"}, "gemma-2b": {"hf_name": "google/gemma-2b", "pretty_name": "Gemma-2B"}, "phi-2": {"hf_name": "microsoft/phi-2", "pretty_name": "Phi-2 @hf"}, "olmo": {"hf_name": "allenai/OLMo-7B", "pretty_name": "OLMo-7B @hf"}, "phi-2-vllm": {"hf_name": "microsoft/phi-2", "pretty_name": "Phi-2 (2.7B)"}, "olmo-7b-vllm": {"hf_name": "allenai/OLMo-7B", "pretty_name": "OLMo-7B"}, "falcon-7b": {"hf_name": "microsoft/falcon-7b", "pretty_name": "Falcon-7B"}, "mpt-7b": {"hf_name": "mosaicml/mpt-7b", "pretty_name": "MPT-7B"}, "amber": {"hf_name": "LLM360/Amber", "pretty_name": "Amber (7B)"}, "dbrx": {"hf_name": "databricks/dbrx-base", "pretty_name": "DBRX-base"}, } def formatter(x): x = round(x, 2) return x def make_clickable_model(model_name, model_info): if model_info[model_name]['hf_name'].startswith("http"): link = model_info[model_name]['hf_name'] else: link = f"https://huggingface.co/{model_info[model_name]['hf_name']}" if model_name.startswith("gpt"): return f'{model_info[model_name]["pretty_name"]}' else: return f'{model_info[model_name]["pretty_name"]}' def build_demo(original_df, full_df, TYPES): with gr.Blocks(theme=gr.themes.Soft(), css=css) as demo: # gr.HTML(BANNER, elem_id="banner") gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") with gr.Tabs(elem_classes="tab-buttons") as tabs: with gr.TabItem("🏅 Leaderboard", elem_id="od-benchmark-tab-table", id=0): leaderboard_table = gr.components.Dataframe( value=original_df, datatype=TYPES, height=1000, wrap=False, elem_id="leaderboard-table", interactive=False, visible=True, min_width=60, ) with gr.TabItem("🐑 URIAL + 🤗 OpenLLM", elem_id="od-benchmark-tab-table", id=1): gr.Markdown("### More results from the awesome 🤗 [Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard) ", elem_classes="markdown-text") leaderboard_table_full = gr.components.Dataframe( value=full_df, datatype=TYPES, height=1000, wrap=False, elem_id="leaderboard-table-full", interactive=False, visible=True, min_width=60, ) gr.Markdown(f"Last updated on **{LAST_UPDATED}**", elem_classes="markdown-text-small") with gr.Row(): with gr.Accordion("📙 Citation", open=False): gr.Textbox( value=CITATION_TEXT, lines=18, label="Copy the BibTeX to cite URIAL and MT-Bench", elem_id="citation-button", show_copy_button=True) # ).style(show_copy_button=True) return demo if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--share", action="store_true") parser.add_argument("--result_file", help="Path to results table", default="leaderboard_data.jsonl") args = parser.parse_args() all_model_hf_ids = {v["hf_name"]: k for k, v in model_info.items()} # Load Open LLM Leaderboard with open("open-llm-leaderboard.json") as f: open_llm_leaderbaord = json.load(f) full_leaderboard = {} for item in open_llm_leaderbaord: if item["Model"] in all_model_hf_ids: # print(item["Model"]) # print(item["Average \u2b06\ufe0f"]) full_bench_item = {} # full_bench_item["hf_name"] = item["Model"] full_bench_item["model_name"] = all_model_hf_ids[item["Model"]] tasks = ["HellaSwag", "ARC", "Winogrande", "TruthfulQA", "MMLU", "GSM8K"] for task in tasks: full_bench_item[task] = item[task] full_bench_item["HF_AVG"] = item["Average \u2b06\ufe0f"] full_leaderboard[all_model_hf_ids[item["Model"]]] = full_bench_item # Load URIAL Leaderboard with open("leaderboard_data.jsonl") as f: for line in f: item = json.loads(line) if item["model"] in full_leaderboard: full_leaderboard[item["model"]]["URIAL_AVG"] = item["Overall"] # Process the URIAL Benchmark Tab original_df = pd.read_json(args.result_file, lines=True) print(original_df.columns) for col in original_df.columns: if col == "model": original_df[col] = original_df[col].apply(lambda x: x.replace(x, make_clickable_model(x, model_info))) else: original_df[col] = original_df[col].apply(formatter) # For numerical values # Define the first column explicitly, add 'Overall' as the second column, and then append the rest excluding 'Overall' new_order = [original_df.columns[0], 'Overall'] + [col for col in original_df.columns if col not in [original_df.columns[0], 'Overall']] # Reorder the DataFrame columns using the new order reordered_df = original_df[new_order] reordered_df.sort_values(by='Overall', inplace=True, ascending=False) reordered_df.rename(columns=column_names, inplace=True) # Process the Full Benchmark Tab full_df = pd.DataFrame(full_leaderboard).T full_df = full_df.reset_index() full_df.rename(columns={"index": "model"}, inplace=True) full_df = full_df[["model", "URIAL_AVG", "HF_AVG", "HellaSwag", "ARC", "Winogrande", "TruthfulQA", "MMLU", "GSM8K"]] full_df.sort_values(by='URIAL_AVG', inplace=True, ascending=False) full_df["model"] = full_df["model"].apply(lambda x: make_clickable_model(x, model_info)) full_df.rename(columns=column_names, inplace=True) # apply formatter to numerical columns for col in full_df.columns: if col not in ["Model"]: full_df[col] = full_df[col].apply(formatter) # For numerical values # COLS = [c.name for c in fields(AutoEvalColumn)] # TYPES = [c.type for c in fields(AutoEvalColumn)] TYPES = ["markdown", "number"] demo = build_demo(reordered_df, full_df, TYPES) demo.launch(share=args.share)