Merge branch 'main' of https://huggingface.co/spaces/bigcode/multilingual-code-evals into main
Browse files- app.py +129 -11
- requirements.txt +3 -1
- src/text_content.py +28 -2
- src/utils.py +37 -1
app.py
CHANGED
@@ -1,15 +1,30 @@
|
|
1 |
# some code blocks are taken from https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/tree/main
|
|
|
|
|
|
|
|
|
2 |
import gradio as gr
|
3 |
import pandas as pd
|
|
|
4 |
|
5 |
from src.css_html import custom_css
|
6 |
-
from src.text_content import ABOUT_TEXT, SUBMISSION_TEXT
|
7 |
-
from src.utils import (
|
8 |
-
|
9 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
10 |
|
|
|
|
|
11 |
df = pd.read_csv("data/code_eval_board.csv")
|
12 |
|
|
|
|
|
13 |
COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
|
14 |
TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
|
15 |
COLS_LITE = [
|
@@ -20,6 +35,65 @@ TYPES_LITE = [
|
|
20 |
]
|
21 |
|
22 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
23 |
def select_columns(df, columns):
|
24 |
always_here_cols = [
|
25 |
AutoEvalColumn.model_type_symbol.name,
|
@@ -56,8 +130,9 @@ with demo:
|
|
56 |
"""<div style="text-align: center;"><h1> ⭐ Big <span style='color: #e6b800;'>Code</span> Models <span style='color: #e6b800;'>Leaderboard</span></h1></div>\
|
57 |
<br>\
|
58 |
<p>Inspired from the <a href="https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard">🤗 Open LLM Leaderboard</a> and <a href="https://huggingface.co/spaces/optimum/llm-perf-leaderboard">🤗 Open LLM-Perf Leaderboard 🏋️</a>, we compare performance of base multilingual code generation models on <a href="https://huggingface.co/datasets/openai_humaneval">HumanEval</a> benchmark and <a href="https://huggingface.co/datasets/nuprl/MultiPL-E">MultiPL-E</a>. We also measure throughput and provide\
|
59 |
-
information about the models. We only compare open pre-trained multilingual code models, that people can start from as base models for their trainings.</p>"""
|
60 |
-
|
|
|
61 |
|
62 |
with gr.Tabs(elem_classes="tab-buttons") as tabs:
|
63 |
with gr.Column():
|
@@ -142,13 +217,16 @@ with demo:
|
|
142 |
[hidden_leaderboard_df, shown_columns],
|
143 |
leaderboard_df,
|
144 |
)
|
145 |
-
gr.Markdown(
|
|
|
146 |
**Notes:**
|
147 |
- Win Rate represents how often a model outperforms other models in each language, averaged across all languages.
|
148 |
- The scores of instruction-tuned models might be significantly higher on humaneval-python than other languages because we use the instruction prompt format of this benchmark.
|
149 |
- For more details check the 📝 About section.
|
150 |
-
""",
|
151 |
-
|
|
|
|
|
152 |
with gr.TabItem("📊 Performance Plot", id=1):
|
153 |
with gr.Row():
|
154 |
bs_1_plot = gr.components.Plot(
|
@@ -161,11 +239,51 @@ with demo:
|
|
161 |
elem_id="bs50-plot",
|
162 |
show_label=False,
|
163 |
)
|
164 |
-
gr.Markdown(
|
|
|
|
|
|
|
165 |
with gr.TabItem("📝 About", id=2):
|
166 |
gr.Markdown(ABOUT_TEXT, elem_classes="markdown-text")
|
167 |
with gr.TabItem("Submit results 🚀", id=3):
|
168 |
gr.Markdown(SUBMISSION_TEXT)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
169 |
|
170 |
|
171 |
-
demo.launch()
|
|
|
1 |
# some code blocks are taken from https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/tree/main
|
2 |
+
import json
|
3 |
+
import os
|
4 |
+
from datetime import datetime, timezone
|
5 |
+
|
6 |
import gradio as gr
|
7 |
import pandas as pd
|
8 |
+
from huggingface_hub import HfApi
|
9 |
|
10 |
from src.css_html import custom_css
|
11 |
+
from src.text_content import ABOUT_TEXT, SUBMISSION_TEXT, SUBMISSION_TEXT_2
|
12 |
+
from src.utils import (
|
13 |
+
AutoEvalColumn,
|
14 |
+
fields,
|
15 |
+
is_model_on_hub,
|
16 |
+
make_clickable_names,
|
17 |
+
plot_throughput,
|
18 |
+
styled_error,
|
19 |
+
styled_message,
|
20 |
+
)
|
21 |
|
22 |
+
TOKEN = os.environ.get("HF_TOKEN", None)
|
23 |
+
api = HfApi(TOKEN)
|
24 |
df = pd.read_csv("data/code_eval_board.csv")
|
25 |
|
26 |
+
QUEUE_REPO = "bigcode/evaluation-requests"
|
27 |
+
EVAL_REQUESTS_PATH = "eval-queue"
|
28 |
COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
|
29 |
TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
|
30 |
COLS_LITE = [
|
|
|
35 |
]
|
36 |
|
37 |
|
38 |
+
def add_new_eval(
|
39 |
+
model: str,
|
40 |
+
revision: str,
|
41 |
+
precision: str,
|
42 |
+
model_type: str,
|
43 |
+
):
|
44 |
+
precision = precision
|
45 |
+
current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
46 |
+
|
47 |
+
if model_type is None or model_type == "":
|
48 |
+
return styled_error("Please select a model type.")
|
49 |
+
|
50 |
+
# check the model actually exists before adding the eval
|
51 |
+
if revision == "":
|
52 |
+
revision = "main"
|
53 |
+
|
54 |
+
model_on_hub, error = is_model_on_hub(model, revision)
|
55 |
+
if not model_on_hub:
|
56 |
+
return styled_error(f'Model "{model}" {error}')
|
57 |
+
|
58 |
+
print("adding new eval")
|
59 |
+
|
60 |
+
eval_entry = {
|
61 |
+
"model": model,
|
62 |
+
"revision": revision,
|
63 |
+
"precision": precision,
|
64 |
+
"status": "PENDING",
|
65 |
+
"submitted_time": current_time,
|
66 |
+
"model_type": model_type.split(" ")[1],
|
67 |
+
}
|
68 |
+
|
69 |
+
user_name = ""
|
70 |
+
model_path = model
|
71 |
+
if "/" in model:
|
72 |
+
user_name = model.split("/")[0]
|
73 |
+
model_path = model.split("/")[1]
|
74 |
+
|
75 |
+
OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
|
76 |
+
os.makedirs(OUT_DIR, exist_ok=True)
|
77 |
+
out_path = f"{OUT_DIR}/{model_path}_eval_request_{precision}.json"
|
78 |
+
print(f"Saving eval request to {out_path}")
|
79 |
+
|
80 |
+
with open(out_path, "w") as f:
|
81 |
+
f.write(json.dumps(eval_entry))
|
82 |
+
|
83 |
+
api.upload_file(
|
84 |
+
path_or_fileobj=out_path,
|
85 |
+
path_in_repo=out_path.split("eval-queue/")[1],
|
86 |
+
repo_id=QUEUE_REPO,
|
87 |
+
repo_type="dataset",
|
88 |
+
commit_message=f"Add {model} to eval queue",
|
89 |
+
)
|
90 |
+
|
91 |
+
# remove the local file
|
92 |
+
os.remove(out_path)
|
93 |
+
|
94 |
+
return styled_message("Your request has been submitted to the evaluation queue!\n")
|
95 |
+
|
96 |
+
|
97 |
def select_columns(df, columns):
|
98 |
always_here_cols = [
|
99 |
AutoEvalColumn.model_type_symbol.name,
|
|
|
130 |
"""<div style="text-align: center;"><h1> ⭐ Big <span style='color: #e6b800;'>Code</span> Models <span style='color: #e6b800;'>Leaderboard</span></h1></div>\
|
131 |
<br>\
|
132 |
<p>Inspired from the <a href="https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard">🤗 Open LLM Leaderboard</a> and <a href="https://huggingface.co/spaces/optimum/llm-perf-leaderboard">🤗 Open LLM-Perf Leaderboard 🏋️</a>, we compare performance of base multilingual code generation models on <a href="https://huggingface.co/datasets/openai_humaneval">HumanEval</a> benchmark and <a href="https://huggingface.co/datasets/nuprl/MultiPL-E">MultiPL-E</a>. We also measure throughput and provide\
|
133 |
+
information about the models. We only compare open pre-trained multilingual code models, that people can start from as base models for their trainings.</p>""",
|
134 |
+
elem_classes="markdown-text",
|
135 |
+
)
|
136 |
|
137 |
with gr.Tabs(elem_classes="tab-buttons") as tabs:
|
138 |
with gr.Column():
|
|
|
217 |
[hidden_leaderboard_df, shown_columns],
|
218 |
leaderboard_df,
|
219 |
)
|
220 |
+
gr.Markdown(
|
221 |
+
"""
|
222 |
**Notes:**
|
223 |
- Win Rate represents how often a model outperforms other models in each language, averaged across all languages.
|
224 |
- The scores of instruction-tuned models might be significantly higher on humaneval-python than other languages because we use the instruction prompt format of this benchmark.
|
225 |
- For more details check the 📝 About section.
|
226 |
+
""",
|
227 |
+
elem_classes="markdown-text",
|
228 |
+
)
|
229 |
+
|
230 |
with gr.TabItem("📊 Performance Plot", id=1):
|
231 |
with gr.Row():
|
232 |
bs_1_plot = gr.components.Plot(
|
|
|
239 |
elem_id="bs50-plot",
|
240 |
show_label=False,
|
241 |
)
|
242 |
+
gr.Markdown(
|
243 |
+
"**Note:** Zero throughput on the right plot refers to OOM, for more details check the 📝 About section.",
|
244 |
+
elem_classes="markdown-text",
|
245 |
+
)
|
246 |
with gr.TabItem("📝 About", id=2):
|
247 |
gr.Markdown(ABOUT_TEXT, elem_classes="markdown-text")
|
248 |
with gr.TabItem("Submit results 🚀", id=3):
|
249 |
gr.Markdown(SUBMISSION_TEXT)
|
250 |
+
gr.Markdown(
|
251 |
+
"## 📤 Submit your model here:", elem_classes="markdown-text"
|
252 |
+
)
|
253 |
+
with gr.Column():
|
254 |
+
with gr.Row():
|
255 |
+
model_name = gr.Textbox(label="Model name")
|
256 |
+
revision_name = gr.Textbox(
|
257 |
+
label="revision", placeholder="main"
|
258 |
+
)
|
259 |
+
with gr.Row():
|
260 |
+
precision = gr.Dropdown(
|
261 |
+
choices=[
|
262 |
+
"float16",
|
263 |
+
"bfloat16",
|
264 |
+
"8bit",
|
265 |
+
"4bit",
|
266 |
+
],
|
267 |
+
label="Precision",
|
268 |
+
multiselect=False,
|
269 |
+
value="float16",
|
270 |
+
interactive=True,
|
271 |
+
)
|
272 |
+
model_type = gr.Dropdown(
|
273 |
+
choices=["🟢 base", "🔶 instruction-tuned"],
|
274 |
+
label="Model type",
|
275 |
+
multiselect=False,
|
276 |
+
value=None,
|
277 |
+
interactive=True,
|
278 |
+
)
|
279 |
+
submit_button = gr.Button("Submit Eval")
|
280 |
+
submission_result = gr.Markdown()
|
281 |
+
submit_button.click(
|
282 |
+
add_new_eval,
|
283 |
+
inputs=[model_name, revision_name, precision, model_type],
|
284 |
+
outputs=[submission_result],
|
285 |
+
)
|
286 |
+
gr.Markdown(SUBMISSION_TEXT_2)
|
287 |
|
288 |
|
289 |
+
demo.launch()
|
requirements.txt
CHANGED
@@ -1 +1,3 @@
|
|
1 |
-
plotly
|
|
|
|
|
|
1 |
+
plotly
|
2 |
+
transformers==4.32.1
|
3 |
+
huggingface-hub==0.16.4
|
src/text_content.py
CHANGED
@@ -29,9 +29,35 @@ The growing number of code models released by the community necessitates a compr
|
|
29 |
|
30 |
SUBMISSION_TEXT = """
|
31 |
<h1 align="center">
|
32 |
-
How to submit
|
33 |
</h1>
|
34 |
-
We welcome the community to submit evaluation results of new models.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
35 |
|
36 |
### 1 - Running Evaluation
|
37 |
|
|
|
29 |
|
30 |
SUBMISSION_TEXT = """
|
31 |
<h1 align="center">
|
32 |
+
How to submit models/results to the leaderboard?
|
33 |
</h1>
|
34 |
+
We welcome the community to submit evaluation results of new models. We also provide an experiental feature for submitting models that our team will evaluate on the 🤗 cluster.
|
35 |
+
|
36 |
+
## Submitting Models (experimental feature)
|
37 |
+
Inspired from the Open LLM Leaderboard, we welcome code models submission from the community that will be automatically evaluated. Please note that this is still an experimental feature.
|
38 |
+
Below are some guidlines to follow before submitting your model:
|
39 |
+
|
40 |
+
#### 1) Make sure you can load your model and tokenizer using AutoClasses:
|
41 |
+
```python
|
42 |
+
from transformers import AutoConfig, AutoModel, AutoTokenizer
|
43 |
+
config = AutoConfig.from_pretrained("your model name", revision=revision)
|
44 |
+
model = AutoModel.from_pretrained("your model name", revision=revision)
|
45 |
+
tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
|
46 |
+
```
|
47 |
+
If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
|
48 |
+
Note: make sure your model is public!
|
49 |
+
Note: if your model needs `use_remote_code=True`, we do not support this option yet.
|
50 |
+
#### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
|
51 |
+
It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
|
52 |
+
#### 3) Make sure your model has an open license!
|
53 |
+
This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
|
54 |
+
#### 4) Fill up your model card
|
55 |
+
When we add extra information about models to the leaderboard, it will be automatically taken from the model card.
|
56 |
+
"""
|
57 |
+
|
58 |
+
SUBMISSION_TEXT_2 = """
|
59 |
+
## Sumbitting Results
|
60 |
+
You also have the option for running evaluation yourself and submitting results. These results will be added as non-verified, the authors are however required to upload their generations in case other members want to check.
|
61 |
|
62 |
### 1 - Running Evaluation
|
63 |
|
src/utils.py
CHANGED
@@ -1,7 +1,7 @@
|
|
1 |
# source: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/blob/main/src/utils_display.py
|
2 |
from dataclasses import dataclass
|
3 |
import plotly.graph_objects as go
|
4 |
-
|
5 |
|
6 |
# These classes are for user facing column names, to avoid having to change them
|
7 |
# all around the code when a modif is needed
|
@@ -113,3 +113,39 @@ def plot_throughput(df, bs=1):
|
|
113 |
yaxis_title="Average Code Score",
|
114 |
)
|
115 |
return fig
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
# source: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/blob/main/src/utils_display.py
|
2 |
from dataclasses import dataclass
|
3 |
import plotly.graph_objects as go
|
4 |
+
from transformers import AutoConfig
|
5 |
|
6 |
# These classes are for user facing column names, to avoid having to change them
|
7 |
# all around the code when a modif is needed
|
|
|
113 |
yaxis_title="Average Code Score",
|
114 |
)
|
115 |
return fig
|
116 |
+
|
117 |
+
|
118 |
+
def styled_error(error):
|
119 |
+
return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
|
120 |
+
|
121 |
+
|
122 |
+
def styled_warning(warn):
|
123 |
+
return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
|
124 |
+
|
125 |
+
|
126 |
+
def styled_message(message):
|
127 |
+
return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
|
128 |
+
|
129 |
+
|
130 |
+
def has_no_nan_values(df, columns):
|
131 |
+
return df[columns].notna().all(axis=1)
|
132 |
+
|
133 |
+
|
134 |
+
def has_nan_values(df, columns):
|
135 |
+
return df[columns].isna().any(axis=1)
|
136 |
+
|
137 |
+
|
138 |
+
def is_model_on_hub(model_name: str, revision: str) -> bool:
|
139 |
+
try:
|
140 |
+
AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=False)
|
141 |
+
return True, None
|
142 |
+
|
143 |
+
except ValueError:
|
144 |
+
return (
|
145 |
+
False,
|
146 |
+
"needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
|
147 |
+
)
|
148 |
+
|
149 |
+
except Exception as e:
|
150 |
+
print(f"Could not get the model config from the hub.: {e}")
|
151 |
+
return False, "was not found on hub!"
|