|
import gradio as gr |
|
import pandas as pd |
|
from huggingface_hub import list_models |
|
import plotly.express as px |
|
|
|
def get_plots(task): |
|
|
|
task_df= pd.read_csv('data/energy/'+task) |
|
params_df = pd.read_csv('data/params/'+task) |
|
params_df= params_df.rename(columns={"Link": "model"}) |
|
all_df = pd.merge(task_df, params_df, on='model') |
|
all_df['Total GPU Energy (Wh)'] = all_df['total_gpu_energy']*1000 |
|
all_df = all_df.sort_values(by=['Total GPU Energy (Wh)']) |
|
all_df['parameters'] = all_df['parameters'].apply(format_params) |
|
all_df['energy_star'] = pd.cut(all_df['Total GPU Energy (Wh)'], 3, labels=["βββ", "ββ", "β"]) |
|
fig = px.scatter(all_df, x="model", y='Total GPU Energy (Wh)', custom_data=['parameters'], height= 500, width= 800, color = 'energy_star', color_discrete_map={"β": 'red', "ββ": "yellow", "βββ": "green"}) |
|
fig.update_traces( |
|
hovertemplate="<br>".join([ |
|
"Total Energy: %{y}", |
|
"Parameters: %{customdata[0]}"]) |
|
) |
|
return fig |
|
|
|
def make_link(mname): |
|
link = "["+ str(mname).split('/')[1] +'](https://huggingface.co/'+str(mname)+")" |
|
return link |
|
|
|
def get_model_names(task_data): |
|
|
|
task_df= pd.read_csv('data/params/'+task_data) |
|
energy_df= pd.read_csv('data/energy/'+task_data) |
|
task_df= task_df.rename(columns={"Link": "model"}) |
|
all_df = pd.merge(task_df, energy_df, on='model') |
|
all_df=all_df.drop_duplicates(subset=['model']) |
|
all_df['parameters'] = all_df['parameters'].apply(format_params) |
|
all_df['model'] = all_df['model'].apply(make_link) |
|
all_df['Total GPU Energy (Wh)'] = all_df['total_gpu_energy']*1000 |
|
all_df['Total GPU Energy (Wh)'] = all_df['Total GPU Energy (Wh)'].round(2) |
|
model_names = all_df[['model','parameters','Total GPU Energy (Wh)']] |
|
return model_names |
|
|
|
def format_params(num): |
|
if num > 1000000000: |
|
if not num % 1000000000: |
|
return f'{num // 1000000000}B' |
|
return f'{round(num / 1000000000, 1)}B' |
|
return f'{num // 1000000}M' |
|
|
|
|
|
|
|
demo = gr.Blocks() |
|
|
|
with demo: |
|
gr.Markdown( |
|
"""# Energy Star Leaderboard - v.0 (2024) π π» π |
|
### Welcome to the leaderboard for the [AI Energy Star Project!](https://huggingface.co/EnergyStarAI) |
|
Click through the tasks below to see how different models measure up in terms of energy efficiency""" |
|
) |
|
gr.Markdown( |
|
"""Test your own models via the [submission portal (TODO)].""" |
|
) |
|
with gr.Tabs(): |
|
with gr.TabItem("Text Generation π¬"): |
|
with gr.Row(): |
|
with gr.Column(): |
|
plot = gr.Plot(get_plots('text_generation.csv')) |
|
with gr.Column(): |
|
table = gr.Dataframe(get_model_names('text_generation.csv'), datatype="markdown") |
|
|
|
with gr.TabItem("Image Generation π·"): |
|
with gr.Row(): |
|
with gr.Column(): |
|
plot = gr.Plot(get_plots('image_generation.csv')) |
|
with gr.Column(): |
|
table = gr.Dataframe(get_model_names('image_generation.csv'), datatype="markdown") |
|
|
|
with gr.TabItem("Text Classification π"): |
|
with gr.Row(): |
|
with gr.Column(): |
|
plot = gr.Plot(get_plots('text_classification.csv')) |
|
with gr.Column(): |
|
table = gr.Dataframe(get_model_names('text_classification.csv'), datatype="markdown") |
|
|
|
with gr.TabItem("Image Classification πΌοΈ"): |
|
with gr.Row(): |
|
with gr.Column(): |
|
plot = gr.Plot(get_plots('image_classification.csv')) |
|
with gr.Column(): |
|
table = gr.Dataframe(get_model_names('image_classification.csv'), datatype="markdown") |
|
|
|
with gr.TabItem("Image Captioning π"): |
|
with gr.Row(): |
|
with gr.Column(): |
|
plot = gr.Plot(get_plots('question_answering.csv')) |
|
with gr.Column(): |
|
table = gr.Dataframe(get_model_names('question_answering.csv'), datatype="markdown") |
|
with gr.TabItem("Summarization π"): |
|
with gr.Row(): |
|
with gr.Column(): |
|
plot = gr.Plot(get_plots('summarization.csv')) |
|
with gr.Column(): |
|
table = gr.Dataframe(get_model_names('summarization.csv'), datatype="markdown") |
|
|
|
with gr.TabItem("Automatic Speech Recognition π¬ "): |
|
with gr.Row(): |
|
with gr.Column(): |
|
plot = gr.Plot(get_plots('asr.csv')) |
|
with gr.Column(): |
|
table = gr.Dataframe(get_model_names('asr.csv'), datatype="markdown") |
|
|
|
with gr.TabItem("Object Detection π"): |
|
with gr.Row(): |
|
with gr.Column(): |
|
plot = gr.Plot(get_plots('object_detection.csv')) |
|
with gr.Column(): |
|
table = gr.Dataframe(get_model_names('object_detection.csv'), datatype="markdown") |
|
|
|
with gr.TabItem("Sentence Similarity π"): |
|
with gr.Row(): |
|
with gr.Column(): |
|
plot = gr.Plot(get_plots('sentence_similarity.csv')) |
|
with gr.Column(): |
|
table = gr.Dataframe(get_model_names('sentence_similarity.csv'), datatype="markdown") |
|
|
|
with gr.TabItem("Extractive QA β"): |
|
with gr.Row(): |
|
with gr.Column(): |
|
plot = gr.Plot(get_plots('question_answering.csv')) |
|
with gr.Column(): |
|
table = gr.Dataframe(get_model_names('question_answering.csv'), datatype="markdown") |
|
with gr.Accordion("Methodology"): |
|
gr.Markdown( |
|
"""For each of the ten tasks above, we created a custom dataset with 1,000 entries (see all of the datasets on our [org Hub page](https://huggingface.co/EnergyStarAI)). |
|
We then tested each of the models from the leaderboard on the appropriate task, measuring the energy consumed using [Code Carbon](https://mlco2.github.io/codecarbon/), an open-source Python package for tracking the environmental impacts of code. |
|
We developed and used a [Docker container](https://github.com/huggingface/EnergyStarAI/) to maximize the reproducibility of results, and to enable members of the community to benchmark internal models. |
|
Reach out to us if you want to collaborate! |
|
""") |
|
demo.launch() |
|
|