Spaces:
Running
Running
import gradio as gr | |
import pandas as pd | |
from src.about import CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, INTRODUCTION_TEXT, LLM_BENCHMARKS_TEXT, SUBMIT_TEXT, TITLE | |
from src.display.css_html_js import custom_css | |
from src.display.utils import COLS, TYPES, AutoEvalColumn, fields | |
from src.envs import CRM_RESULTS_PATH | |
from src.populate import get_leaderboard_df_crm | |
original_df = get_leaderboard_df_crm(CRM_RESULTS_PATH, COLS) | |
leaderboard_df = original_df.copy() | |
# leaderboard_df = leaderboard_df.style.format({"accuracy_metric_average": "{0:.2f}"}) | |
# Searching and filtering | |
def update_table( | |
hidden_df: pd.DataFrame, | |
columns: list, | |
framework_query: list | |
# llm_query: list, | |
# llm_provider_query: list, | |
# accuracy_method_query: str, | |
# accuracy_threshold_query: str, | |
# use_case_area_query: list, | |
# use_case_query: list, | |
# use_case_type_query: list, | |
# metric_area_query: list, | |
): | |
filtered_df = filter_framework_func(hidden_df, framework_query) | |
# filtered_df = filter_llm_func(hidden_df, llm_query) | |
# filtered_df = filter_llm_provider_func(filtered_df, llm_provider_query) | |
# filtered_df = filter_accuracy_method_func(filtered_df, accuracy_method_query) | |
# filtered_df["Accuracy Threshold"] = filter_accuracy_threshold_func(filtered_df, accuracy_threshold_query) | |
# filtered_df = filtered_df[filtered_df["Accuracy Threshold"]] | |
# filtered_df["Use Case Area"] = filtered_df["Use Case Name"].apply(lambda x: x.split(": ")[0]) | |
# filtered_df = filter_use_case_area_func(filtered_df, use_case_area_query) | |
# filtered_df = filter_use_case_func(filtered_df, use_case_query) | |
# filtered_df = filter_use_case_type_func(filtered_df, use_case_type_query) | |
# Filtering by metric area | |
# metric_area_maps = { | |
# "Cost": ["Cost Band"], | |
# "Accuracy": ["Accuracy", "Instruction Following", "Conciseness", "Completeness", "Factuality"], | |
# "Speed (Latency)": ["Response Time (Sec)", "Mean Output Tokens"], | |
# "Trust & Safety": ["Trust & Safety", "Safety", "Privacy", "Truthfulness", "CRM Fairness"], | |
# } | |
# all_metric_cols = [] | |
# for area in metric_area_maps: | |
# all_metric_cols = all_metric_cols + metric_area_maps[area] | |
# columns_to_keep = list(set(columns).difference(set(all_metric_cols))) | |
# for area in metric_area_query: | |
# columns_to_keep = columns_to_keep + metric_area_maps[area] | |
# columns = list(set(columns).intersection(set(columns_to_keep))) | |
df = select_columns(filtered_df, columns) | |
return df.style.map(highlight_cost_band_low, props="background-color: #b3d5a4") | |
# def highlight_cols(x): | |
# df = x.copy() | |
# df.loc[:, :] = "color: black" | |
# df.loc[, ["Accuracy"]] = "background-color: #b3d5a4" | |
# return df | |
def highlight_cost_band_low(s, props=""): | |
return props if s == "Low" else None | |
def init_leaderboard_df( | |
leaderboard_df: pd.DataFrame, | |
columns: list, | |
llm_query: list, | |
# llm_provider_query: list, | |
# accuracy_method_query: str, | |
# accuracy_threshold_query: str, | |
# use_case_area_query: list, | |
# use_case_query: list, | |
# use_case_type_query: list, | |
# metric_area_query: list, | |
): | |
# Applying the style function | |
# return df.style.apply(highlight_cols, axis=None) | |
return update_table( | |
leaderboard_df, | |
columns, | |
llm_query, | |
# llm_provider_query, | |
# accuracy_method_query, | |
# accuracy_threshold_query, | |
# use_case_area_query, | |
# use_case_query, | |
# use_case_type_query, | |
# metric_area_query, | |
) | |
def filter_accuracy_method_func(df: pd.DataFrame, accuracy_method_query: str) -> pd.DataFrame: | |
return df[df["Accuracy Method"] == accuracy_method_query] | |
def filter_accuracy_threshold_func(df: pd.DataFrame, accuracy_threshold_query: str) -> pd.DataFrame: | |
accuracy_cols = ["Instruction Following", "Conciseness", "Completeness", "Accuracy"] | |
return (df.loc[:, accuracy_cols] >= float(accuracy_threshold_query)).all(axis=1) | |
def filter_use_case_area_func(df: pd.DataFrame, use_case_area_query: list) -> pd.DataFrame: | |
return df[ | |
df["Use Case Area"].apply( | |
lambda x: len(set([_.strip() for _ in x.split("&")]).intersection(use_case_area_query)) | |
) | |
> 0 | |
] | |
def filter_use_case_func(df: pd.DataFrame, use_case_query: list) -> pd.DataFrame: | |
return df[df["Use Case Name"].isin(use_case_query)] | |
def filter_use_case_type_func(df: pd.DataFrame, use_case_type_query: list) -> pd.DataFrame: | |
return df[df["Use Case Type"].isin(use_case_type_query)] | |
def filter_llm_func(df: pd.DataFrame, llm_query: list) -> pd.DataFrame: | |
return df[df["Model"].isin(llm_query)] | |
def filter_framework_func(df: pd.DataFrame, framework_query: list) -> pd.DataFrame: | |
return df[df["Agentic Framework"].isin(framework_query)] | |
def filter_llm_provider_func(df: pd.DataFrame, llm_provider_query: list) -> pd.DataFrame: | |
return df[df["LLM Provider"].isin(llm_provider_query)] | |
def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame: | |
# always_here_cols = [ | |
# AutoEvalColumn.model.name, | |
# ] | |
# model_provider_col = [AutoEvalColumn.model_provider.name] if AutoEvalColumn.model_provider.name in columns else [] | |
# We use COLS to maintain sortingx | |
filtered_df = df[ | |
( | |
[AutoEvalColumn.model.name] | |
# + model_provider_col | |
+ [AutoEvalColumn.agentic_framework.name] | |
+ [c for c in COLS if c in df.columns and c in columns ] | |
+ [AutoEvalColumn.overall.name] | |
) | |
] | |
return filtered_df | |
demo = gr.Blocks(css=custom_css) | |
with demo: | |
gr.HTML(TITLE) | |
gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") | |
with gr.Tabs(elem_classes="tab-buttons") as tabs: | |
with gr.TabItem("π CRMArena Benchmark", elem_id="llm-benchmark-tab-table", id=0): | |
with gr.Row(): | |
with gr.Column(): | |
filter_agentic_framework = gr.CheckboxGroup( | |
choices=list(original_df["Agentic Framework"].unique()), | |
value=list(original_df["Agentic Framework"].unique()), | |
label="Agentic Framework", | |
info="", | |
interactive=True, | |
) | |
with gr.Row(): | |
shown_columns = gr.CheckboxGroup( | |
choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden], | |
value=[ | |
c.name | |
for c in fields(AutoEvalColumn) | |
if c.displayed_by_default and not c.hidden and not c.never_hidden | |
], | |
label="Select tasks to show", | |
elem_id="column-select", | |
interactive=True, | |
) | |
# with gr.Column(): | |
# filter_llm = gr.CheckboxGroup( | |
# choices=list(original_df["Model"].unique()), | |
# value=list(original_df["Model"].unique()), | |
# label="Model", | |
# info="", | |
# interactive=True, | |
# ) | |
# with gr.Column(): | |
# with gr.Row(): | |
# filter_llm_provider = gr.CheckboxGroup( | |
# choices=list(original_df["LLM Provider"].unique()), | |
# value=list(original_df["LLM Provider"].unique()), | |
# label="LLM Provider", | |
# info="", | |
# interactive=True, | |
# ) | |
# with gr.Row(): | |
# filter_metric_area = gr.CheckboxGroup( | |
# choices=["Accuracy", "Speed (Latency)", "Trust & Safety", "Cost"], | |
# value=["Accuracy", "Speed (Latency)", "Trust & Safety", "Cost"], | |
# label="Metric Area", | |
# info="", | |
# interactive=True, | |
# ) | |
# with gr.Row(): | |
# filter_use_case = gr.CheckboxGroup( | |
# choices=list(original_df["Use Case Name"].unique()), | |
# value=list(original_df["Use Case Name"].unique()), | |
# label="Use Case", | |
# info="", | |
# # multiselect=True, | |
# interactive=True, | |
# ) | |
# with gr.Row(): | |
# with gr.Column(): | |
# filter_use_case_area = gr.CheckboxGroup( | |
# choices=["Service", "Sales"], | |
# value=["Service", "Sales"], | |
# label="Use Case Area", | |
# info="", | |
# interactive=True, | |
# ) | |
# with gr.Column(): | |
# filter_use_case_type = gr.CheckboxGroup( | |
# choices=["Summary", "Generation"], | |
# value=["Summary", "Generation"], | |
# label="Use Case Type", | |
# info="", | |
# interactive=True, | |
# ) | |
# with gr.Column(): | |
# filter_use_case = gr.Dropdown( | |
# choices=list(original_df["Use Case Name"].unique()), | |
# value=list(original_df["Use Case Name"].unique()), | |
# label="Use Case", | |
# info="", | |
# multiselect=True, | |
# interactive=True, | |
# ) | |
# with gr.Column(): | |
# filter_accuracy_method = gr.Radio( | |
# choices=["Manual", "Auto"], | |
# value="Manual", | |
# label="Accuracy Method", | |
# info="", | |
# interactive=True, | |
# ) | |
# with gr.Column(): | |
# filter_accuracy_threshold = gr.Number( | |
# value="0", | |
# label="Accuracy Threshold", | |
# info="Range: 0.0 to 4.0", | |
# interactive=True, | |
# ) | |
leaderboard_table = gr.components.Dataframe( | |
# value=leaderboard_df[[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value], | |
value=init_leaderboard_df( | |
leaderboard_df, | |
shown_columns.value, | |
filter_agentic_framework.value | |
# filter_llm.value, | |
# filter_llm_provider.value, | |
# filter_accuracy_method.value, | |
# filter_accuracy_threshold.value, | |
# filter_use_case_area.value, | |
# filter_use_case.value, | |
# filter_use_case_type.value, | |
# filter_metric_area.value, | |
), | |
headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value, | |
datatype=TYPES, | |
elem_id="leaderboard-table", | |
interactive=False, | |
visible=True, | |
) | |
# Dummy leaderboard for handling the case when the user uses backspace key | |
hidden_leaderboard_table_for_search = gr.components.Dataframe( | |
value=original_df[COLS], | |
headers=COLS, | |
datatype=TYPES, | |
visible=False, | |
) | |
for selector in [ | |
shown_columns, | |
filter_agentic_framework | |
# filter_llm, | |
# filter_llm_provider, | |
# filter_accuracy_method, | |
# filter_accuracy_threshold, | |
# filter_use_case_area, | |
# filter_use_case, | |
# filter_use_case_type, | |
# filter_metric_area, | |
]: | |
selector.change( | |
update_table, | |
[ | |
hidden_leaderboard_table_for_search, | |
shown_columns, | |
filter_agentic_framework, | |
# filter_llm, | |
# filter_llm_provider, | |
# filter_accuracy_method, | |
# filter_accuracy_threshold, | |
# filter_use_case_area, | |
# filter_use_case, | |
# filter_use_case_type, | |
# filter_metric_area, | |
], | |
leaderboard_table, | |
queue=True, | |
) | |
with gr.TabItem("π About", elem_id="llm-benchmark-tab-table", id=3): | |
gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text") | |
with gr.TabItem("π Submit", elem_id="llm-benchmark-tab-table", id=4): | |
gr.Markdown(SUBMIT_TEXT, elem_classes="markdown-text") | |
with gr.Row(): | |
with gr.Accordion("π Citation", open=False): | |
citation_button = gr.Textbox( | |
value=CITATION_BUTTON_TEXT, | |
label=CITATION_BUTTON_LABEL, | |
lines=20, | |
elem_id="citation-button", | |
show_copy_button=True, | |
) | |
# scheduler = BackgroundScheduler() | |
# scheduler.add_job(restart_space, "interval", seconds=1800) | |
# scheduler.start() | |
demo.queue(default_concurrency_limit=40).launch() | |