import gradio as gr import pandas as pd import os import zipfile import base64 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results" CITATION_BUTTON_TEXT = r"""@misc{aienergyscore-leaderboard, author = {Sasha Luccioni and Boris Gamazaychikov and Emma Strubell and Sara Hooker and Yacine Jernite and Carole-Jean Wu and Margaret Mitchell}, title = {AI Energy Score Leaderboard - February 2025}, year = {2025}, publisher = {Hugging Face}, howpublished = "\url{https://huggingface.co/spaces/AIEnergyScore/Leaderboard}", }""" # List of tasks (CSV filenames) tasks = [ 'asr.csv', 'object_detection.csv', 'text_classification.csv', 'image_captioning.csv', 'question_answering.csv', 'text_generation.csv', 'image_classification.csv', 'sentence_similarity.csv', 'image_generation.csv', 'summarization.csv' ] ### HELPER FUNCTIONS ### def format_stars(score): try: score_int = int(score) except Exception: score_int = 0 return f'{"★" * score_int}' def make_link(mname): parts = str(mname).split('/') display_name = parts[1] if len(parts) > 1 else mname return f'{display_name}' def extract_link_text(html_link): start = html_link.find('>') + 1 end = html_link.rfind('') if start > 0 and end > start: return html_link[start:end] else: return html_link def generate_html_table_from_df(df): # Compute a static width for the Model column based on the longest model name. if not df.empty: max_length = max(len(extract_link_text(link)) for link in df['Model']) else: max_length = 10 static_width = max_length * 10 + 16 max_energy = df['gpu_energy_numeric'].max() if not df.empty else 1 color_map = {"1": "black", "2": "black", "3": "black", "4": "black", "5": "black"} html = '' html += '' html += '' html += '' html += '' html += '' html += '' html += '' for _, row in df.iterrows(): energy_numeric = row['gpu_energy_numeric'] energy_str = f"{energy_numeric:,.2f}" bar_width = (energy_numeric / max_energy) * 100 score_val = row['energy_score'] bar_color = color_map.get(str(score_val), "gray") html += '' html += f'' html += f'' html += (f'') html += f'' html += '' html += '
ModelProviderGPU Energy (Wh)Score
{row["Model"]}{row["Provider"]}{energy_str}
' f'
{row["Score"]}
' return f'
{html}
' def process_df(task, sort_order="Low to High", filter_fn=None): df = pd.read_csv(os.path.join("data", "energy", task)) if df.columns[0].startswith("Unnamed:"): df = df.iloc[:, 1:] df['energy_score'] = df['energy_score'].astype(int) df['gpu_energy_numeric'] = pd.to_numeric(df['total_gpu_energy'], errors='raise') * 1000 if filter_fn is not None: df = filter_fn(df) df['Provider'] = df['model'].apply(lambda x: str(x).split('/')[0]) df['Model'] = df['model'].apply(make_link) df['Score'] = df['energy_score'].apply(format_stars) ascending = True if sort_order == "Low to High" else False df = df.sort_values(by='gpu_energy_numeric', ascending=ascending) return df def compute_efficiency_ratio(df): if df.empty: return 1 min_val = df['gpu_energy_numeric'].min() max_val = df['gpu_energy_numeric'].max() ratio = max_val / min_val if min_val > 0 else 1 return ratio def generate_info_callout(ratio, scope_text): """ Returns a "did you know" callout with a lightbulb emoji. The callout uses a light green background, a small font, and is limited to a max-width of 250px. It is wrapped in a container that aligns it to the right. """ return ( f'
' f'
' f'💡 There\'s a {ratio:,.1f}x difference between the highest and lowest energy use in {scope_text}.' f'
' ) def get_global_callout(): all_df = pd.DataFrame() for task in tasks: df = pd.read_csv(os.path.join("data", "energy", task)) if df.columns[0].startswith("Unnamed:"): df = df.iloc[:, 1:] df['gpu_energy_numeric'] = pd.to_numeric(df['total_gpu_energy'], errors='raise') * 1000 all_df = pd.concat([all_df, df], ignore_index=True) ratio = compute_efficiency_ratio(all_df) return generate_info_callout(ratio, "this leaderboard") ### ZIP DOWNLOAD FUNCTIONS ### def zip_csv_files(): data_dir = os.path.join("data", "energy") zip_filename = "data.zip" with zipfile.ZipFile(zip_filename, "w", zipfile.ZIP_DEFLATED) as zipf: for filename in os.listdir(data_dir): if filename.endswith(".csv"): filepath = os.path.join(data_dir, filename) zipf.write(filepath, arcname=filename) return zip_filename def get_zip_data_link(): zip_filename = zip_csv_files() with open(zip_filename, "rb") as f: data = f.read() b64 = base64.b64encode(data).decode() href = ( f'Download Data' ) return href ### UPDATE FUNCTIONS (RETURNING CALLOUT AND TABLE HTML) ### def update_text_generation(selected_display, sort_order): mapping = { "A (Single Consumer GPU) <20B parameters": "A", "B (Single Cloud GPU) 20-66B parameters": "B", "C (Multiple Cloud GPUs) >66B parameters": "C" } model_class = mapping.get(selected_display, "A") def filter_fn(df): if 'class' in df.columns: return df[df['class'] == model_class] return df df = process_df('text_generation.csv', sort_order, filter_fn) ratio = compute_efficiency_ratio(df) # For Text Generation, use "this class" as the scope. callout = generate_info_callout(ratio, "this class") table_html = generate_html_table_from_df(df) return callout, table_html def update_image_generation(sort_order): df = process_df('image_generation.csv', sort_order) ratio = compute_efficiency_ratio(df) callout = generate_info_callout(ratio, "this task") table_html = generate_html_table_from_df(df) return callout, table_html def update_text_classification(sort_order): df = process_df('text_classification.csv', sort_order) ratio = compute_efficiency_ratio(df) callout = generate_info_callout(ratio, "this task") table_html = generate_html_table_from_df(df) return callout, table_html def update_image_classification(sort_order): df = process_df('image_classification.csv', sort_order) ratio = compute_efficiency_ratio(df) callout = generate_info_callout(ratio, "this task") table_html = generate_html_table_from_df(df) return callout, table_html def update_image_captioning(sort_order): df = process_df('image_captioning.csv', sort_order) ratio = compute_efficiency_ratio(df) callout = generate_info_callout(ratio, "this task") table_html = generate_html_table_from_df(df) return callout, table_html def update_summarization(sort_order): df = process_df('summarization.csv', sort_order) ratio = compute_efficiency_ratio(df) callout = generate_info_callout(ratio, "this task") table_html = generate_html_table_from_df(df) return callout, table_html def update_asr(sort_order): df = process_df('asr.csv', sort_order) ratio = compute_efficiency_ratio(df) callout = generate_info_callout(ratio, "this task") table_html = generate_html_table_from_df(df) return callout, table_html def update_object_detection(sort_order): df = process_df('object_detection.csv', sort_order) ratio = compute_efficiency_ratio(df) callout = generate_info_callout(ratio, "this task") table_html = generate_html_table_from_df(df) return callout, table_html def update_sentence_similarity(sort_order): df = process_df('sentence_similarity.csv', sort_order) ratio = compute_efficiency_ratio(df) callout = generate_info_callout(ratio, "this task") table_html = generate_html_table_from_df(df) return callout, table_html def update_extractive_qa(sort_order): df = process_df('question_answering.csv', sort_order) ratio = compute_efficiency_ratio(df) callout = generate_info_callout(ratio, "this task") table_html = generate_html_table_from_df(df) return callout, table_html def update_all_tasks(sort_order): all_df = pd.DataFrame() for task in tasks: df = pd.read_csv(os.path.join("data", "energy", task)) if df.columns[0].startswith("Unnamed:"): df = df.iloc[:, 1:] df['energy_score'] = df['energy_score'].astype(int) df['gpu_energy_numeric'] = pd.to_numeric(df['total_gpu_energy'], errors='raise') * 1000 df['Provider'] = df['model'].apply(lambda x: str(x).split('/')[0]) df['Model'] = df['model'].apply(make_link) df['Score'] = df['energy_score'].apply(format_stars) all_df = pd.concat([all_df, df], ignore_index=True) all_df = all_df.drop_duplicates(subset=['model']) ascending = True if sort_order == "Low to High" else False all_df = all_df.sort_values(by='gpu_energy_numeric', ascending=ascending) ratio = compute_efficiency_ratio(all_df) callout = generate_info_callout(ratio, "this leaderboard") table_html = generate_html_table_from_df(all_df) return callout, table_html ### GLOBAL HEADER (Logo & Global Callout) ### # Use a element so that dark mode uses logodark.png. global_header_html = f"""
Logo
{get_global_callout()}
""" ### CUSTOM CSS for Dark Mode and Mobile Responsiveness ### custom_css = """ /* Table and layout */ .data-table { table-layout: fixed; width: 100%; } .data-table th, .data-table td { max-width: 150px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } .table-container { width: 100%; margin-left: auto; margin-right: auto; } /* Force header links to be black in light mode */ .header-link { color: black !important; } /* Dark mode styles */ @media (prefers-color-scheme: dark) { body { background-color: #121212; color: #e0e0e0; } .data-table thead { background-color: #333; } .data-table th { color: #e0e0e0; } .data-table td { color: #e0e0e0; } /* Non-header links in dark mode */ a:not(.header-link) { color: #3fa45bff !important; } } /* Mobile styles: hide callout boxes on small screens */ @media (max-width: 600px) { .info-callout { display: none !important; } } """ ### GRADIO INTERFACE ### demo = gr.Blocks(css=custom_css) with demo: # --- Header Links --- gr.HTML(f"""
Submission Portal Label Generator FAQ Documentation {get_zip_data_link()} Community
""") # --- Global Header: Centered Logo with Global Callout at Right Edge --- gr.HTML(global_header_html) # --- Tabs for the different tasks --- with gr.Tabs(): # --- Text Generation Tab --- with gr.TabItem("Text Generation 💬"): with gr.Row(): with gr.Column(scale=4): model_class_options = [ "A (Single Consumer GPU) <20B parameters", "B (Single Cloud GPU) 20-66B parameters", "C (Multiple Cloud GPUs) >66B parameters" ] model_class_dropdown = gr.Dropdown(choices=model_class_options, label="Select Model Class", value=model_class_options[0]) with gr.Column(scale=4): sort_dropdown_tg = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") with gr.Column(scale=4): tg_callout = gr.HTML() tg_table = gr.HTML() init_callout, init_table = update_text_generation(model_class_options[0], "Low to High") tg_callout.value = init_callout tg_table.value = init_table model_class_dropdown.change(fn=update_text_generation, inputs=[model_class_dropdown, sort_dropdown_tg], outputs=[tg_callout, tg_table]) sort_dropdown_tg.change(fn=update_text_generation, inputs=[model_class_dropdown, sort_dropdown_tg], outputs=[tg_callout, tg_table]) # --- Image Generation Tab --- with gr.TabItem("Image Generation 📷"): with gr.Row(): with gr.Column(scale=8): sort_dropdown_img = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") with gr.Column(scale=4): img_callout = gr.HTML() img_table = gr.HTML() init_callout, init_table = update_image_generation("Low to High") img_callout.value = init_callout img_table.value = init_table sort_dropdown_img.change(fn=update_image_generation, inputs=sort_dropdown_img, outputs=[img_callout, img_table]) # --- Text Classification Tab --- with gr.TabItem("Text Classification 🎭"): with gr.Row(): with gr.Column(scale=8): sort_dropdown_tc = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") with gr.Column(scale=4): tc_callout = gr.HTML() tc_table = gr.HTML() init_callout, init_table = update_text_classification("Low to High") tc_callout.value = init_callout tc_table.value = init_table sort_dropdown_tc.change(fn=update_text_classification, inputs=sort_dropdown_tc, outputs=[tc_callout, tc_table]) # --- Image Classification Tab --- with gr.TabItem("Image Classification 🖼️"): with gr.Row(): with gr.Column(scale=8): sort_dropdown_ic = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") with gr.Column(scale=4): ic_callout = gr.HTML() ic_table = gr.HTML() init_callout, init_table = update_image_classification("Low to High") ic_callout.value = init_callout ic_table.value = init_table sort_dropdown_ic.change(fn=update_image_classification, inputs=sort_dropdown_ic, outputs=[ic_callout, ic_table]) # --- Image Captioning Tab --- with gr.TabItem("Image Captioning 📝"): with gr.Row(): with gr.Column(scale=8): sort_dropdown_icap = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") with gr.Column(scale=4): icap_callout = gr.HTML() icap_table = gr.HTML() init_callout, init_table = update_image_captioning("Low to High") icap_callout.value = init_callout icap_table.value = init_table sort_dropdown_icap.change(fn=update_image_captioning, inputs=sort_dropdown_icap, outputs=[icap_callout, icap_table]) # --- Summarization Tab --- with gr.TabItem("Summarization 📃"): with gr.Row(): with gr.Column(scale=8): sort_dropdown_sum = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") with gr.Column(scale=4): sum_callout = gr.HTML() sum_table = gr.HTML() init_callout, init_table = update_summarization("Low to High") sum_callout.value = init_callout sum_table.value = init_table sort_dropdown_sum.change(fn=update_summarization, inputs=sort_dropdown_sum, outputs=[sum_callout, sum_table]) # --- Automatic Speech Recognition Tab --- with gr.TabItem("Automatic Speech Recognition 💬"): with gr.Row(): with gr.Column(scale=8): sort_dropdown_asr = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") with gr.Column(scale=4): asr_callout = gr.HTML() asr_table = gr.HTML() init_callout, init_table = update_asr("Low to High") asr_callout.value = init_callout asr_table.value = init_table sort_dropdown_asr.change(fn=update_asr, inputs=sort_dropdown_asr, outputs=[asr_callout, asr_table]) # --- Object Detection Tab --- with gr.TabItem("Object Detection 🚘"): with gr.Row(): with gr.Column(scale=8): sort_dropdown_od = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") with gr.Column(scale=4): od_callout = gr.HTML() od_table = gr.HTML() init_callout, init_table = update_object_detection("Low to High") od_callout.value = init_callout od_table.value = init_table sort_dropdown_od.change(fn=update_object_detection, inputs=sort_dropdown_od, outputs=[od_callout, od_table]) # --- Sentence Similarity Tab --- with gr.TabItem("Sentence Similarity 📚"): with gr.Row(): with gr.Column(scale=8): sort_dropdown_ss = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") with gr.Column(scale=4): ss_callout = gr.HTML() ss_table = gr.HTML() init_callout, init_table = update_sentence_similarity("Low to High") ss_callout.value = init_callout ss_table.value = init_table sort_dropdown_ss.change(fn=update_sentence_similarity, inputs=sort_dropdown_ss, outputs=[ss_callout, ss_table]) # --- Extractive QA Tab --- with gr.TabItem("Extractive QA ❔"): with gr.Row(): with gr.Column(scale=8): sort_dropdown_qa = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") with gr.Column(scale=4): qa_callout = gr.HTML() qa_table = gr.HTML() init_callout, init_table = update_extractive_qa("Low to High") qa_callout.value = init_callout qa_table.value = init_table sort_dropdown_qa.change(fn=update_extractive_qa, inputs=sort_dropdown_qa, outputs=[qa_callout, qa_table]) # --- All Tasks Tab --- with gr.TabItem("All Tasks 💡"): with gr.Row(): with gr.Column(scale=8): sort_dropdown_all = gr.Dropdown(choices=["Low to High", "High to Low"], label="Sort", value="Low to High") with gr.Column(scale=4): all_callout = gr.HTML() all_table = gr.HTML() init_callout, init_table = update_all_tasks("Low to High") all_callout.value = init_callout all_table.value = init_table sort_dropdown_all.change(fn=update_all_tasks, inputs=sort_dropdown_all, outputs=[all_callout, all_table]) with gr.Accordion("📙 Citation", open=False): citation_button = gr.Textbox( value=CITATION_BUTTON_TEXT, label=CITATION_BUTTON_LABEL, elem_id="citation-button", lines=10, show_copy_button=True, ) gr.Markdown("Last updated: February 2025") demo.launch()