Spaces:
Running
Running
| __all__ = ['block', 'make_clickable_model', 'make_clickable_user', 'get_submissions'] | |
| import gradio as gr | |
| import pandas as pd | |
| import re | |
| import pandas as pd | |
| import numpy as np | |
| from collections import defaultdict | |
| from constants import * | |
| import os | |
| from huggingface_hub import Repository | |
| import json | |
| global data_component, filter_component | |
| TOKEN = os.environ.get("TOKEN") | |
| repo = Repository(local_dir="./download_from_dataset", clone_from="JMMMU/leaderboard_result", repo_type="dataset", use_auth_token=TOKEN) | |
| current_directory = os.getcwd() | |
| def validate_model_size(s): | |
| pattern = r'^\d+B$|^-$' | |
| if re.match(pattern, s): | |
| return s | |
| else: | |
| return '-' | |
| def upload_file(files): | |
| file_paths = [file.name for file in files] | |
| return file_paths | |
| def get_acc(data, subject_list): | |
| acc = 0 | |
| for subject in subject_list: | |
| acc += data["results"][subject]['jmmmu_acc,none'] | |
| acc = acc/len(subject_list) | |
| acc = acc * 100 | |
| acc = round(acc, 1) | |
| return acc | |
| def calculate_score(input_file): | |
| json_string = input_file.decode('utf-8') | |
| data = json.loads(json_string) | |
| result_dict = {} | |
| overall = data["results"]["jmmmu"]['jmmmu_acc,none']*100 | |
| ca = data["results"]["culture_agnostic"]['jmmmu_acc,none']*100 | |
| cs = data["results"]["culture_specific"]['jmmmu_acc,none']*100 | |
| overall = round(overall, 1) | |
| ca = round(ca, 1) | |
| cs = round(cs, 1) | |
| # Art_Psychology | |
| art_psychology_subject_list = ["jmmmu_design", "jmmmu_music", "jmmmu_psychology"] | |
| # Science | |
| science_subject_list = ["jmmmu_biology", "jmmmu_chemistry", "jmmmu_physics", "jmmmu_math"] | |
| # Business | |
| business_subject_list = ["jmmmu_accounting", "jmmmu_economics", "jmmmu_finance", "jmmmu_manage", "jmmmu_marketing"] | |
| # Medicine | |
| medicine_subject_list = ["jmmmu_basic_medical_science", "jmmmu_clinical_medicine", "jmmmu_diagnostics_and_laboratory_medicine", "jmmmu_pharmacy", "jmmmu_public_health"] | |
| # Tech_Eng. | |
| tech_eng_subject_list = ["jmmmu_agriculture", "jmmmu_architecture_and_engineering", "jmmmu_computer_science", "jmmmu_electronics", "jmmmu_energy_and_power", "jmmmu_materials", "jmmmu_mechanical_engineering"] | |
| jmmmu_japanese_art_subject_list = ["jmmmu_japanese_art"] | |
| jmmmu_japanese_heritage_subject_list = ["jmmmu_japanese_heritage"] | |
| jmmmu_japanese_history_subject_list = ["jmmmu_japanese_history"] | |
| jmmmu_world_history_subject_list = ["jmmmu_world_history"] | |
| art_psychology = get_acc(data, art_psychology_subject_list) | |
| science = get_acc(data, science_subject_list) | |
| business = get_acc(data, business_subject_list) | |
| medicine = get_acc(data, medicine_subject_list) | |
| tech_eng = get_acc(data, tech_eng_subject_list) | |
| japanese_art = get_acc(data, jmmmu_japanese_art_subject_list) | |
| japanese_heritage = get_acc(data, jmmmu_japanese_heritage_subject_list) | |
| japanese_history = get_acc(data, jmmmu_japanese_history_subject_list) | |
| world_history = get_acc(data, jmmmu_world_history_subject_list) | |
| result_dict =\ | |
| { | |
| "overall": overall, | |
| "cultureSpecific": cs, | |
| "cultureAgnostic": ca, | |
| "japaneseArt": japanese_art, | |
| "japaneseHeritage": japanese_heritage, | |
| "japaneseHistory": japanese_history, | |
| "worldHistory": world_history, | |
| "artPsychology": art_psychology, | |
| "business": business, | |
| "science": science, | |
| "healthMedicine": medicine, | |
| "techEngineering": tech_eng | |
| } | |
| return result_dict | |
| def add_new_eval( | |
| input_file, | |
| model_type: str, | |
| model_name_textbox: str, | |
| revision_name_textbox: str, | |
| model_link: str, | |
| model_size: str, | |
| # upd_type: str, | |
| # question_type: str | |
| ): | |
| if input_file is None: | |
| warning_text = "Error! Empty file!" | |
| print(warning_text) | |
| return warning_text | |
| else: | |
| model_size = validate_model_size(model_size) | |
| # if upd_type == 'AAD': | |
| csv_path = CSV_RESULT_PATH | |
| # validity_check(input_file) | |
| csv_data = pd.read_csv(csv_path) | |
| result_dict = calculate_score(input_file) | |
| if revision_name_textbox == '': | |
| col = csv_data.shape[0] | |
| model_name = model_name_textbox | |
| else: | |
| model_name = revision_name_textbox | |
| model_name_list = csv_data['Model'] | |
| name_list = [name.split(']')[0][1:] for name in model_name_list] | |
| if revision_name_textbox not in name_list: | |
| col = csv_data.shape[0] | |
| else: | |
| col = name_list.index(revision_name_textbox) | |
| model_name_wo_link = model_name | |
| if model_link == '': | |
| model_name = model_name # no url | |
| else: | |
| model_name = '[' + model_name + '](' + model_link + ')' | |
| # add new data | |
| new_data = [ | |
| model_type, | |
| model_name, | |
| model_size, | |
| result_dict["overall"], | |
| result_dict["cultureSpecific"], | |
| result_dict["cultureAgnostic"], | |
| result_dict["japaneseArt"], | |
| result_dict["japaneseHeritage"], | |
| result_dict["japaneseHistory"], | |
| result_dict["worldHistory"], | |
| result_dict["artPsychology"], | |
| result_dict["business"], | |
| result_dict["science"], | |
| result_dict["healthMedicine"], | |
| result_dict["techEngineering"] | |
| ] | |
| # If the same data already exists, return an error. | |
| if new_data in csv_data.values.tolist(): | |
| warning_text = "Error! The same data already exists!" | |
| print(warning_text) | |
| return warning_text | |
| # If the same model name already exists, return an error. | |
| elif new_data[:5] in csv_data.values.tolist(): | |
| warning_text = "Error! The same data already exists! Please fill revision_name." | |
| print(warning_text) | |
| return warning_text | |
| csv_data.loc[col] = new_data | |
| csv_data = csv_data.to_csv(csv_path, index=False) | |
| absolute_result_path = os.path.abspath(csv_path) | |
| if not os.path.exists(absolute_result_path): | |
| raise FileNotFoundError(f"File {absolute_result_path} not found") | |
| repo.git_pull() | |
| repo.git_add(absolute_result_path) | |
| save_path = os.path.join(CSV_QUEUE_DIR, f"{model_name_wo_link}.json") | |
| with open(save_path, "wb") as f: | |
| f.write(input_file) | |
| absolute_queue_path = os.path.abspath(save_path) | |
| repo.git_add(absolute_queue_path) | |
| repo.git_commit(f"add {model_name_wo_link} results") | |
| repo.git_push() | |
| print(f"Success! Your {model_name_wo_link} has been added!") | |
| return 0 | |
| def get_baseline_df(): | |
| repo.git_pull() | |
| df = pd.read_csv(CSV_RESULT_PATH) | |
| df = df.sort_values(by="Overall", ascending=False) | |
| present_columns = MODEL_INFO + checkbox_group.value | |
| df = df[present_columns] | |
| return df | |
| def get_all_df(): | |
| repo.git_pull() | |
| df = pd.read_csv(CSV_RESULT_PATH) | |
| df = df.sort_values(by="Overall", ascending=False) | |
| return df | |
| block = gr.Blocks() | |
| with block: | |
| gr.Markdown( | |
| LEADERBORAD_INTRODUCTION | |
| ) | |
| with gr.Tabs(elem_classes="tab-buttons") as tabs: | |
| # table jmmmu bench | |
| with gr.TabItem("🏅 JMMMU Benchmark", elem_id="jmmmu-benchmark-tab-table", id=1): | |
| # selection for column part: | |
| checkbox_group = gr.CheckboxGroup( | |
| choices=TASK_INFO, | |
| value=AVG_INFO, | |
| label="Evaluation Dimension", | |
| interactive=True, | |
| ) # user can select the evaluation dimension | |
| with gr.Row(): | |
| # selection for model size part: | |
| model_size = gr.CheckboxGroup( | |
| choices=MODEL_SIZE, | |
| value=MODEL_SIZE, | |
| label="Model Size", | |
| interactive=True, | |
| ) | |
| baseline_value = get_baseline_df() | |
| baseline_header = MODEL_INFO + checkbox_group.value | |
| baseline_datatype = ['markdown'] * 2 + ['number'] * len(checkbox_group.value) | |
| data_component = gr.components.Dataframe( | |
| value=baseline_value, | |
| headers=baseline_header, | |
| type="pandas", | |
| datatype=baseline_datatype, | |
| interactive=False, | |
| visible=True, | |
| ) | |
| def on_filter_model_size_method_change(selected_model_size, selected_columns): | |
| updated_data = get_all_df() | |
| # model_size | |
| def custom_filter(row, model_size_filters): | |
| model_size = row['Model Size'] | |
| model_size = model_size.upper() | |
| if model_size == '-': | |
| size_filter = '-' in model_size_filters | |
| elif 'B' in model_size: | |
| size = float(model_size.replace('B', '')) | |
| size_filter = ('>=10B' in model_size_filters and size >= 10) or ('<10B' in model_size_filters and size < 10) | |
| else: | |
| size_filter = False | |
| return size_filter | |
| mask = updated_data.apply(custom_filter, axis=1, model_size_filters=selected_model_size) | |
| updated_data = updated_data[mask] | |
| # columns: | |
| selected_columns = [item for item in TASK_INFO if item in selected_columns] | |
| present_columns = MODEL_INFO + selected_columns | |
| updated_data = updated_data[present_columns] | |
| updated_data = updated_data.sort_values(by=selected_columns[0], ascending=False) | |
| updated_headers = present_columns | |
| update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers] | |
| filter_component = gr.components.Dataframe( | |
| value=updated_data, | |
| headers=updated_headers, | |
| type="pandas", | |
| datatype=update_datatype, | |
| interactive=False, | |
| visible=True, | |
| ) | |
| return filter_component | |
| model_size.change(fn=on_filter_model_size_method_change, inputs=[model_size, checkbox_group], outputs=data_component) | |
| checkbox_group.change(fn=on_filter_model_size_method_change, inputs=[model_size, checkbox_group], outputs=data_component) | |
| # table 5 | |
| with gr.TabItem("🚀 Submit here! ", elem_id="jmmmu-benchmark-tab-table", id=5): | |
| with gr.Row(): | |
| gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text") | |
| with gr.Row(): | |
| gr.Markdown("# ✉️✨ Submit your model evaluation json file here!", elem_classes="markdown-text") | |
| with gr.Row(): | |
| with gr.Column(): | |
| model_type = gr.Dropdown( | |
| choices=["LMM", "LLM"], | |
| label="Model type", | |
| multiselect=False, | |
| value="LMM", | |
| interactive=True, | |
| ) | |
| model_name_textbox = gr.Textbox( | |
| label="Model name", placeholder="LLaMA-7B" | |
| ) | |
| revision_name_textbox = gr.Textbox( | |
| label="Revision Model Name", placeholder="LLaMA-7B" | |
| ) | |
| model_link = gr.Textbox( | |
| label="Model Link", placeholder="https://huggingface.co/decapoda-research/llama-7b-hf" | |
| ) | |
| model_size = gr.Textbox( | |
| label="Model size", placeholder="7B(Input content format must be 'number+B' or '-', default is '-')" | |
| ) | |
| with gr.Column(): | |
| input_file = gr.components.File(label="Click to Upload a JSON File", file_count="single", type='binary') | |
| submit_button = gr.Button("Submit Eval") | |
| submission_result = gr.Markdown() | |
| submit_button.click( | |
| add_new_eval, | |
| inputs = [ | |
| input_file, | |
| model_type, | |
| model_name_textbox, | |
| revision_name_textbox, | |
| model_link, | |
| model_size | |
| ], | |
| ) | |
| def refresh_data(): | |
| value = get_baseline_df() | |
| return value | |
| with gr.Row(): | |
| data_run = gr.Button("Refresh") | |
| data_run.click( | |
| refresh_data, outputs=[data_component] | |
| ) | |
| with gr.Accordion("Citation", open=False): | |
| citation_button = gr.Textbox( | |
| value=CITATION_BUTTON_TEXT, | |
| label=CITATION_BUTTON_LABEL, | |
| elem_id="citation-button", | |
| show_copy_button=True, | |
| ) | |
| block.launch() | |