Spaces:
Running
Running
| import gradio as gr | |
| import pandas as pd | |
| import os | |
| import plotly.express as px | |
| import numpy as np | |
| datadir = 'data/emissions/complete' | |
| seq2seq_finetuned = ['sshleifer/distilbart-xsum-12-6', 'sshleifer/distilbart-cnn-12-6', 'sshleifer/distilbart-cnn-6-6', | |
| 'pszemraj/led-large-book-summary', 'google/pegasus-xsum', 'google/pegasus-large', | |
| 'google/pegasus-multi_news' ,'facebook/bart-large-cnn', 'ainize/bart-base-cnn'] | |
| color_discrete_map = {'Task-specific Encoder': '#636EFA', 'Multi-purpose Seq2Seq': '#AB63FA', 'Multi-purpose Decoder': '#00CC96', 'Task-specific Seq2Seq':'#EF553B'} | |
| def multi_check(mname): | |
| if 'flan' in mname: | |
| return 'Seq2Seq' | |
| elif 'bloomz' in mname: | |
| return 'Decoder' | |
| def encoder_check(mname): | |
| if 'flan' in mname: | |
| return 'Multi-purpose Seq2Seq' | |
| elif mname in seq2seq_finetuned: | |
| return 'Task-specific Seq2Seq' | |
| elif 'bloomz' in mname: | |
| return 'Multi-purpose Decoder' | |
| else: | |
| return 'Task-specific Encoder' | |
| # Data loading | |
| model_param_df = pd.read_csv('data/model_parameters.csv', header=0) | |
| model_performance_df = pd.read_csv('data/performance.csv', header=0) | |
| emissions_df = pd.read_csv('data/co2_data.csv',header=0) | |
| modalities_df = pd.read_csv('data/modalities_data.csv',header=0) | |
| finetuned_df = emissions_df[~emissions_df['task'].str.contains('zero')] | |
| finetuned_df['task'] = finetuned_df['task'].str.replace('_',' ') | |
| zeroshot_df = emissions_df[emissions_df['task'].str.contains('zero')] | |
| zeroshot_df['task'] = zeroshot_df['task'].str.replace('_',' ') | |
| zeroshot_df['architecture_type'] = zeroshot_df.apply(lambda x : multi_check(x.model),axis=1) | |
| grouped_df = emissions_df.groupby(['model','task']).mean() | |
| grouped_df = grouped_df.reset_index() | |
| grouped_df = grouped_df.drop('task',axis=1) | |
| performance_all = pd.merge(grouped_df, model_performance_df, on='model') | |
| performance_all['type']= performance_all.apply(lambda x : encoder_check(x.model),axis=1) | |
| performance_all['log_emissions'] = np.log1p(performance_all["query emissions (g)"]) | |
| sent_df = performance_all[['imdb (acc)','sst2 (acc)','tomatoes (acc)', "query emissions (g)", 'model','type','num_params', 'log_emissions']][performance_all['task'].isin(['sentiment'])] | |
| qa_df = performance_all[['sciq (acc)', 'squad (f1)', 'squad_v2 (f1, has answer)', "query emissions (g)", 'model','type','num_params', 'log_emissions']][performance_all['task'].isin(['qa'])] | |
| summ_df = performance_all[['samsum (rouge)', 'xsum (rouge)', 'cnn (rouge)', "query emissions (g)", 'model','type', 'num_params','log_emissions']][performance_all['task'].isin(['summarization'])] | |
| # Figure loading | |
| fig0 = px.scatter(emissions_df, x="num_params", y="query emissions (g)", color="model", log_x=True, log_y=True) | |
| fig0.update_layout(xaxis={'categoryorder':'mean ascending'}) | |
| fig0.update_layout(yaxis_title='Total carbon emitted (g)') | |
| fig0.update_layout(xaxis_title='Number of Parameters') | |
| fig1 = px.scatter(finetuned_df, x="task", y="query_energy (kWh)", color="model", log_y=True) | |
| fig1.update_layout(xaxis={'categoryorder':'mean ascending'}) | |
| fig1.update_layout(yaxis_title='Total energy used (Wh)') | |
| fig1.update_layout(xaxis_title='Task') | |
| fig2 = px.scatter(modalities_df, x="num_params", y="query emissions (g)", color="modality", | |
| log_x=True, log_y=True, custom_data=['model','task']) | |
| fig2.update_layout(xaxis_title='Model size (number of parameters)') | |
| fig2.update_layout(yaxis_title='Model emissions (g of CO<sub>2</sub>)') | |
| fig3 = px.scatter(zeroshot_df, x="model", y="query emissions (g)", color="architecture_type", size='num_params', log_y=True) | |
| fig3.update_layout(xaxis={'categoryorder':'mean ascending'}) | |
| fig3.update_layout(yaxis_title='Model emissions (g of CO<sub>2</sub>)') | |
| fig3.update_layout(xaxis_title='Model') | |
| fig4 = px.scatter(zeroshot_df, x="dataset", y="query emissions (g)", color="model", size='num_params', log_y=True) | |
| fig4.update_layout(xaxis={'categoryorder':'mean ascending'}) | |
| fig4.update_layout(yaxis_title='Model emissions (g of CO<sub>2</sub>)') | |
| fig4.update_layout(xaxis_title='Model') | |
| fig5 = px.scatter(sent_df, y=['imdb (acc)', 'sst2 (acc)', 'tomatoes (acc)'], x="num_params", color="type", color_discrete_map=color_discrete_map, | |
| size= "log_emissions", log_x=True, hover_data="model") | |
| fig5.update_layout(legend=dict(y=-0.4,x=0.3)) | |
| fig5.update_layout(yaxis_title='Text Classification Accuracy') | |
| fig6 = px.scatter(qa_df, y=['sciq (acc)', 'squad (f1)', 'squad_v2 (f1, has answer)'], x="num_params", color="type", | |
| size = 'log_emissions', log_x=True, hover_data="model") | |
| fig6.update_layout(legend=dict(y=-0.4,x=0.3)) | |
| fig6.update_layout(yaxis_title='QA accuracy/F1') | |
| fig7 = px.scatter(summ_df, y=['samsum (rouge)', 'xsum (rouge)', 'cnn (rouge)'], x="num_params", color="type", | |
| size = 'log_emissions', log_x=True, hover_data="model") | |
| fig7.update_layout(legend=dict(y=-0.4,x=0.3)) | |
| fig7.update_layout(yaxis_title='Summarization Rouge Score') | |
| demo = gr.Blocks() | |
| with demo: | |
| gr.Markdown("# CO2 Inference Demo 🌎 💻 ⚡") | |
| gr.Markdown("### TL;DR - We ran a series of experiments to measure the energy efficiency and carbon emissions of different\ | |
| models from the HuggingFace Hub, and to see how different tasks and models compare.\ | |
| We found that multi-purpose, generative models are orders of magnitude more energy-intensive than task-specific systems\ | |
| for a variety of tasks, even for models with a similar number of parameters") | |
| gr.Markdown("### Explore the plots below to get more insights about the different models and tasks from our study.") | |
| with gr.Accordion("More details about our methodology:", open=False): | |
| gr.Markdown("We chose ten ML tasks: text classification, token classification, question answering, \ | |
| ), masked language modeling, text generation, summarization, image classification, object detection, \ | |
| image captioning and image generation. For each of the taks, we chose three of the most downloaded datasets and 8 of the most \ | |
| downloaded models from the Hugging Face Hub. We ran each of the models ten times over a 1,000 sample from each of the models and measured the energy consumed and carbon emitted.") | |
| with gr.Row(): | |
| with gr.Column(): | |
| gr.Markdown("## All models from our study (carbon)") | |
| gr.Markdown("### Double click on the model name in the list on the right to isolate its datapoints:") | |
| gr.Markdown("The axes of the plot are in logarithmic scale, meaning that the difference between the least carbon-intensive and the most carbon-intensive models is over 9,000 times!") | |
| gr.Plot(fig0) | |
| with gr.Row(): | |
| with gr.Column(): | |
| gr.Markdown("## Task-by-task comparison (energy)") | |
| gr.Markdown("### Grouping the models by task, we can see different patterns emerge:") | |
| gr.Markdown("Image generation is by far the most energy- and carbon-intensive task from the ones studied, and text classification \ | |
| is the least.") | |
| gr.Plot(fig1) | |
| with gr.Row(): | |
| with gr.Column(): | |
| gr.Markdown("## Modality comparison (carbon)") | |
| gr.Markdown("### Grouping the models by their modality shows different characteristics:") | |
| gr.Markdown("We can see that tasks involving images (image-to-text, image-to-category) require more energy and emit more carbon\ | |
| than ones involving text.") | |
| gr.Plot(fig2) | |
| gr.Markdown("## Multi-task model comparison (carbon)") | |
| gr.Markdown("### Looking at the emissions of multi-task models, we can see that decoder-only models tend to emit more carbon compared to sequence-to-sequence ones.") | |
| gr.Markdown("### This pattern varies depending on the dataset and task - for summarization datasets (the 3 rightmost ones), the difference between models is less obvious.") | |
| with gr.Row(): | |
| with gr.Column(): | |
| gr.Plot(fig3) | |
| with gr.Column(): | |
| gr.Plot(fig4) | |
| gr.Markdown("## Evaluations (accuracy vs carbon)") | |
| gr.Markdown("### Single-task models are, ceteris paribus, less carbon-intensive than multi-task models for all 3 tasks we looked at: ") | |
| with gr.Row(): | |
| with gr.Column(): | |
| gr.Markdown("### Sentiment Analysis") | |
| gr.Plot(fig5) | |
| with gr.Column(): | |
| gr.Markdown("### Question Answering") | |
| gr.Plot(fig6) | |
| with gr.Column(): | |
| gr.Markdown("### Summarization") | |
| gr.Plot(fig7) | |
| demo.launch() | |