| import gradio as gr | |
| import numpy as np | |
| import pandas as pd | |
| df = pd.read_csv("code_eval_board.csv") | |
| df = df.sort_values(by=["Average score"], ascending=False) | |
| headers = [ | |
| "Language", | |
| "Average score", | |
| "Throughput (tokens/s)", | |
| "languages", | |
| "Seq_length", | |
| ] + df.columns.to_list() | |
| demo = gr.Blocks() | |
| with demo: | |
| with gr.Row(): | |
| gr.Markdown( | |
| """<div style="text-align: center;"><h1> ⭐ Base Code Models <span style='color: #e6b800;'>Evaluation</span></h1></div>\ | |
| <br>\ | |
| <p>We compare base code generation models based on <a href="https://huggingface.co/datasets/openai_humaneval">HumanEval</a> benchmark and <a href="https://huggingface.co/datasets/nuprl/MultiPL-E">MultiPL-E</a>, in addition to throughput measurment\ | |
| and information about the modelh. We only compare pre-trained models without instruction tuning.</p>""" | |
| ) | |
| with gr.Column(): | |
| leaderboard_df = gr.components.Dataframe( | |
| value=df, headers=headers, datatype=["str" for _ in range(len(headers))] | |
| ) | |
| with gr.Row(): | |
| gr.Markdown( | |
| """Notes: | |
| <ul> | |
| <li> Average score is the average over all languages, for each model we exclude languages with a score that are less than 1 for the averaging.</li> | |
| <li> Throughputs are measured using <a href="https://github.com/huggingface/optimum-benchmark/tree/main">Optimum-Benchmark</a> with powers <a href="https://huggingface.co/spaces/optimum/llm-perf-leaderboard">LLM Perf LeaderBoard</a>.</li> | |
| <li> HumanEval-Python, reports the pass@1 on HumanEval, the rest is from MultiPL-E benchmark.</li> | |
| <li> All models were evaluated with the <a href="https://github.com/bigcode-project/bigcode-evaluation-harness/tree/main">bigcode-evaluation-harness</a> with top-p=0.95, temperature=0.2 and n_samples=50</li> | |
| </ul>""" | |
| ) | |
| demo.launch() | |