Spaces:
Running
Running
File size: 2,429 Bytes
ab5f5f1 a8a6326 ab5f5f1 0232cf1 ab5f5f1 a8a6326 76b423c a8a6326 76b423c ab5f5f1 76b423c a8a6326 0232cf1 76b423c ab5f5f1 0232cf1 ab5f5f1 0232cf1 a8a6326 0232cf1 ab5f5f1 0232cf1 ab5f5f1 a8a6326 ab5f5f1 0232cf1 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 |
import gradio as gr
from src.utils import model_hyperlink, process_score
LEADERBOARD_COLUMN_TO_DATATYPE = {
# open llm
"Model 🤗": "markdown",
"Experiment 🧪": "str",
# primary measurements
"Prefill (s)": "number",
"Decode (tokens/s)": "number",
"Memory (MB)": "number",
"Energy (tokens/kWh)": "number",
# deployment settings
"Backend 🏭": "str",
"Precision 📥": "str",
"Quantization 🗜️": "str",
"Attention 👁️": "str",
"Kernel ⚛️": "str",
# additional measurements
# "Reserved Memory (MB)": "number",
# "Used Memory (MB)": "number",
"Open LLM Score (%)": "number",
"End-to-End (s)": "number",
"Architecture 🏛️": "str",
"Params (B)": "number",
}
PRIMARY_COLUMNS = [
"Model 🤗",
"Experiment 🧪",
"Prefill (s)",
"Decode (tokens/s)",
"Memory (MB)",
"Energy (tokens/kWh)",
"Open LLM Score (%)",
]
def process_model(model_name):
link = f"https://huggingface.co/{model_name}"
return model_hyperlink(link, model_name)
def get_leaderboard_df(llm_perf_df):
df = llm_perf_df.copy()
# transform for leaderboard
df["Model 🤗"] = df["Model 🤗"].apply(process_model)
# process quantization for leaderboard
df["Open LLM Score (%)"] = df.apply(
lambda x: process_score(x["Open LLM Score (%)"], x["Quantization 🗜️"]),
axis=1,
)
return df
def create_leaderboard_table(llm_perf_df):
# get dataframe
leaderboard_df = get_leaderboard_df(llm_perf_df)
# create search bar
with gr.Row():
search_bar = gr.Textbox(
label="Model 🤗",
info="🔍 Search for a model name",
elem_id="search-bar",
)
# create checkboxes
with gr.Row():
columns_checkboxes = gr.CheckboxGroup(
label="Columns 📊",
value=PRIMARY_COLUMNS,
choices=list(LEADERBOARD_COLUMN_TO_DATATYPE.keys()),
info="☑️ Select the columns to display",
elem_id="columns-checkboxes",
)
# create table
leaderboard_table = gr.components.Dataframe(
value=leaderboard_df[PRIMARY_COLUMNS],
datatype=list(LEADERBOARD_COLUMN_TO_DATATYPE.values()),
headers=list(LEADERBOARD_COLUMN_TO_DATATYPE.keys()),
elem_id="leaderboard-table",
)
return search_bar, columns_checkboxes, leaderboard_table
|