File size: 7,413 Bytes
9203553 e2473e2 cdf268e e2473e2 6c92442 9203553 a9273cf e2473e2 6c92442 e2473e2 9203553 04b315a e2473e2 b7e5123 307c2a1 b7e5123 0ab9033 fd03829 307c2a1 fd03829 e2473e2 6c92442 e2473e2 6c92442 e2473e2 6c92442 e2473e2 6c92442 e2473e2 fd03829 f098e7d fd03829 7471397 fd03829 0ab9033 fd03829 e2473e2 6c92442 e2473e2 96ff3bc e2473e2 6c92442 e2473e2 04b315a e2473e2 2d26479 04f40cd 2d26479 e2473e2 9203553 e2473e2 9203553 e2473e2 9203553 e2473e2 04f40cd e2473e2 6c92442 e2473e2 2d26479 e2473e2 04b315a e2473e2 b7e5123 e2473e2 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 |
import logging
import os
import gradio as gr # type: ignore[import]
from src.content import (
INTRODUCTION_TEXT,
INTRODUCTION_TITLE,
LEADERBOARD_TEXT,
LEADERBOARD_TITLE,
SUBMISSION_TEXT_FILES,
SUBMISSION_TEXT_INTRO,
SUBMISSION_TEXT_METADATA,
SUBMISSION_TEXT_SUBMIT,
SUBMISSION_TEXT_TASK,
SUBMISSION_TITLE,
)
from src.get_results_for_task import get_results_for_task
from src.leaderboard_formatting import get_types_per_task
from src.submission_uploader import SubmissionUploader
from src.tasks_content import (
TASKS_DESCRIPTIONS,
TASKS_PRETTY,
TASKS_PRETTY_REVERSE,
get_submission_text_files_for_task,
)
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s",
handlers=[logging.StreamHandler()],
)
submission_uploader = SubmissionUploader(
dataset_id=os.environ["DATASET_ID"], private_dataset_id=os.environ["PRIVATE_DATASET_ID"]
)
def get_leaderboard_for_task(task_pretty: str) -> gr.components.Dataframe:
return gr.components.Dataframe(
value=get_results_for_task(task_pretty),
interactive=False,
datatype=get_types_per_task(TASKS_PRETTY_REVERSE[task_pretty]),
)
code_completion_dataset_names = get_results_for_task(TASKS_PRETTY['project_code_completion'])['Dataset Name'].unique().tolist()
def get_leaderboard_for_completion_task(dataset_name: str | None):
df = get_results_for_task(TASKS_PRETTY['project_code_completion'])
if dataset_name is None:
dataset_name = code_completion_dataset_names[0]
filtered_df = df[df['Dataset Name']==dataset_name]
return gr.components.Dataframe(
value=filtered_df,
interactive=False,
datatype=get_types_per_task('project_code_completion'),
)
with gr.Blocks() as demo:
# intro
gr.HTML(INTRODUCTION_TITLE)
gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
# leaderboard
gr.HTML(LEADERBOARD_TITLE)
gr.Markdown(LEADERBOARD_TEXT, elem_classes="markdown-text")
with gr.Tabs():
for task_pretty in TASKS_PRETTY_REVERSE:
with gr.TabItem(task_pretty):
with gr.Row():
gr.Markdown(TASKS_DESCRIPTIONS[TASKS_PRETTY_REVERSE[task_pretty]])
if task_pretty == TASKS_PRETTY['project_code_completion']:
leaderboard_table = get_leaderboard_for_completion_task(dataset_name=None)
else:
leaderboard_table = get_leaderboard_for_task(task_pretty)
task_input = gr.Text(value=task_pretty, visible=False)
if task_pretty == TASKS_PRETTY['project_code_completion']:
dataset_dropdown = gr.Dropdown(choices=code_completion_dataset_names, label="Select the Dataset")
dataset_dropdown.change(
fn=get_leaderboard_for_completion_task,
inputs=dataset_dropdown,
outputs=leaderboard_table
)
refresh_button = gr.Button("🔄 Refresh", variant="secondary")
refresh_button.click(
fn=get_leaderboard_for_completion_task,
inputs=dataset_dropdown,
outputs=leaderboard_table,
)
else:
refresh_button = gr.Button("🔄 Refresh", variant="secondary")
refresh_button.click(
fn=get_leaderboard_for_task,
inputs=task_input,
outputs=leaderboard_table,
)
# submission
gr.HTML(SUBMISSION_TITLE)
gr.Markdown(SUBMISSION_TEXT_INTRO, elem_classes="markdown-text")
with gr.Accordion("🚀 Submit new results", open=False):
gr.Markdown(SUBMISSION_TEXT_TASK, elem_classes="markdown-text")
task_selection = gr.Radio(TASKS_PRETTY_REVERSE.keys(), label="Task")
gr.Markdown(SUBMISSION_TEXT_METADATA, elem_classes="markdown-text")
with gr.Row():
with gr.Column():
model_folder_textbox = gr.Textbox(
label="Model Folder",
placeholder="How to call a folder related to this submission in our results dataset (should be unique).",
)
model_name_textbox = gr.Textbox(
label="Model Name",
placeholder="How to display model's name on the leaderboard.",
)
model_url_textbox = gr.Textbox(
label="Model URL",
placeholder="Link to a model's page - will be clickable on a leaderboard (optional).",
)
with gr.Column():
url_textbox = gr.Textbox(
label="Relevant URLs",
placeholder='URLs to relevant resources with additional details about your submission (optional). Use the following format: "[text1](link1), [text2](link2)".',
)
model_availability_textbox = gr.Textbox(
label="Availability",
placeholder="Information about the model's availability and licensing.",
)
context_size_textbox = gr.Textbox(
label="Context Size",
placeholder="Context size in tokens used for the submission (should be an integer).",
)
with gr.Column():
submitted_by_textbox = gr.Textbox(
label="Submitted By",
placeholder="How to display on the leaderboard who submitted the model.",
)
contact_textbox = gr.Textbox(
label="Contact Information",
placeholder="How Long Code Arena team can contact you (won't go to public dataset).",
)
comment_textbox = gr.Textbox(
label="Comment",
placeholder="Any comments you have for Long Code Arena team (optional, won't go to public dataset).",
)
gr.Markdown(SUBMISSION_TEXT_FILES, elem_classes="markdown-text")
with gr.Row():
with gr.Column(variant="panel"):
task_specific_instructions = gr.Markdown(get_submission_text_files_for_task(None))
task_selection.select(get_submission_text_files_for_task, [task_selection], task_specific_instructions)
with gr.Column():
file_output = gr.File(file_count="multiple")
gr.Markdown(SUBMISSION_TEXT_SUBMIT, elem_classes="markdown-text")
submit_button = gr.Button("Submit")
submission_result = gr.Markdown()
submit_button.click(
submission_uploader.upload_files,
[
task_selection,
model_folder_textbox,
model_name_textbox,
model_availability_textbox,
model_url_textbox,
url_textbox,
context_size_textbox,
submitted_by_textbox,
contact_textbox,
comment_textbox,
file_output,
],
submission_result,
)
if __name__ == "__main__":
demo.queue()
demo.launch()
|