File size: 1,942 Bytes
77a9749 caa834f 77a9749 caa834f 77a9749 caa834f 77a9749 caa834f 77a9749 caa834f 77a9749 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 |
import gradio as gr
import numpy as np
import pandas as pd
df = pd.read_csv("code_eval_board.csv")
df = df.sort_values(by=["Average score"], ascending=False)
headers = [
"Language",
"Average score",
"Throughput (tokens/s)",
"languages",
"Seq_length",
] + df.columns.to_list()
demo = gr.Blocks()
with demo:
with gr.Row():
gr.Markdown(
"""<div style="text-align: center;"><h1> ⭐ Base Code Models <span style='color: #e6b800;'>Evaluation</span></h1></div>\
<br>\
<p>We compare base code generation models based on <a href="https://huggingface.co/datasets/openai_humaneval">HumanEval</a> benchmark and <a href="https://huggingface.co/datasets/nuprl/MultiPL-E">MultiPL-E</a>, in addition to throughput measurment\
and information about the modelh. We only compare pre-trained models without instruction tuning.</p>"""
)
with gr.Column():
leaderboard_df = gr.components.Dataframe(
value=df, headers=headers, datatype=["str" for _ in range(len(headers))]
)
with gr.Row():
gr.Markdown(
"""Notes:
<ul>
<li> Average score is the average over all languages, for each model we exclude languages with a score that are less than 1 for the averaging.</li>
<li> Throughputs are measured using <a href="https://github.com/huggingface/optimum-benchmark/tree/main">Optimum-Benchmark</a> with powers <a href="https://huggingface.co/spaces/optimum/llm-perf-leaderboard">LLM Perf LeaderBoard</a>.</li>
<li> HumanEval-Python, reports the pass@1 on HumanEval, the rest is from MultiPL-E benchmark.</li>
<li> All models were evaluated with the <a href="https://github.com/bigcode-project/bigcode-evaluation-harness/tree/main">bigcode-evaluation-harness</a> with top-p=0.95, temperature=0.2 and n_samples=50</li>
</ul>"""
)
demo.launch()
|