category

README.md

@@ -1,14 +1,15 @@
 ---
-title: Chatbot Arena Leaderboard
+title: LMSys Chatbot Arena Leaderboard
 emoji: 🏆🤖
 colorFrom: indigo
 colorTo: green
 sdk: gradio
+sdk_version: 3.50.2
+app_file: app.py
 pinned: false
 license: apache-2.0
 tags:
-- leaderboard
-sdk_version: 4.44.1
+  - leaderboard
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,61 +1,451 @@
-from fastchat.serve.monitor.monitor import build_leaderboard_tab, build_basic_stats_tab, basic_component_values, leader_component_values
-from fastchat.utils import build_logger, get_window_url_params_js
-
+"""A gradio app that renders a static leaderboard. This is used for Hugging Face Space."""
+import ast
 import argparse
 import glob
-import re
+import pickle
+
 import gradio as gr
+import numpy as np
+import pandas as pd
+
+
+# notebook_url = "https://colab.research.google.com/drive/1RAWb22-PFNI-X1gPVzc927SGUdfr6nsR?usp=sharing"
+notebook_url = "https://colab.research.google.com/drive/1KdwokPjirkTmpO_P1WByFNFiqxWQquwH#scrollTo=o_CpbkGEbhrK"
+
+
+basic_component_values = [None] * 6
+leader_component_values = [None] * 5
+
+
+def make_default_md(arena_df, elo_results):
+    total_votes = sum(arena_df["num_battles"]) // 2
+    total_models = len(arena_df)
+
+    leaderboard_md = f"""
+# 🏆 LMSYS Chatbot Arena Leaderboard
+| [Vote](https://chat.lmsys.org) | [Blog](https://lmsys.org/blog/2023-05-03-arena/) | [GitHub](https://github.com/lm-sys/FastChat) | [Paper](https://arxiv.org/abs/2306.05685) | [Dataset](https://github.com/lm-sys/FastChat/blob/main/docs/dataset_release.md) | [Twitter](https://twitter.com/lmsysorg) | [Discord](https://discord.gg/HSWAKCrnFx) |
+
+LMSYS [Chatbot Arena](https://lmsys.org/blog/2023-05-03-arena/) is a crowdsourced open platform for LLM evals.
+We've collected over **500,000** human preference votes to rank LLMs with the Elo ranking system.
+"""
+    return leaderboard_md
+
+
+def make_arena_leaderboard_md(arena_df):
+    total_votes = sum(arena_df["num_battles"]) // 2
+    total_models = len(arena_df)
+
+    leaderboard_md = f"""
+Total #models: **{total_models}**. Total #votes: **{total_votes}**. Last updated: March 29, 2024.
+
+Contribute your vote 🗳️ at [chat.lmsys.org](https://chat.lmsys.org)! Find more analysis in the [notebook]({notebook_url}).
+"""
+    return leaderboard_md
+
+
+def make_full_leaderboard_md(elo_results):
+    leaderboard_md = f"""
+Three benchmarks are displayed: **Arena Elo**, **MT-Bench** and **MMLU**.
+- [Chatbot Arena](https://chat.lmsys.org/?arena) - a crowdsourced, randomized battle platform. We use 500K+ user votes to compute Elo ratings.
+- [MT-Bench](https://arxiv.org/abs/2306.05685): a set of challenging multi-turn questions. We use GPT-4 to grade the model responses.
+- [MMLU](https://arxiv.org/abs/2009.03300) (5-shot): a test to measure a model's multitask accuracy on 57 tasks.
+
+💻 Code: The MT-bench scores (single-answer grading on a scale of 10) are computed by [fastchat.llm_judge](https://github.com/lm-sys/FastChat/tree/main/fastchat/llm_judge).
+The MMLU scores are mostly computed by [InstructEval](https://github.com/declare-lab/instruct-eval).
+Higher values are better for all benchmarks. Empty cells mean not available.
+"""
+    return leaderboard_md
+
+
+def make_leaderboard_md_live(elo_results):
+    leaderboard_md = f"""
+# Leaderboard
+Last updated: {elo_results["last_updated_datetime"]}
+{elo_results["leaderboard_table"]}
+"""
+    return leaderboard_md
+
+
+def update_elo_components(max_num_files, elo_results_file):
+    log_files = get_log_files(max_num_files)
+
+    # Leaderboard
+    if elo_results_file is None:  # Do live update
+        battles = clean_battle_data(log_files)
+        elo_results = report_elo_analysis_results(battles)
+
+        leader_component_values[0] = make_leaderboard_md_live(elo_results)
+        leader_component_values[1] = elo_results["win_fraction_heatmap"]
+        leader_component_values[2] = elo_results["battle_count_heatmap"]
+        leader_component_values[3] = elo_results["bootstrap_elo_rating"]
+        leader_component_values[4] = elo_results["average_win_rate_bar"]
+
+    # Basic stats
+    basic_stats = report_basic_stats(log_files)
+    md0 = f"Last updated: {basic_stats['last_updated_datetime']}"
+
+    md1 = "### Action Histogram\n"
+    md1 += basic_stats["action_hist_md"] + "\n"
+
+    md2 = "### Anony. Vote Histogram\n"
+    md2 += basic_stats["anony_vote_hist_md"] + "\n"
+
+    md3 = "### Model Call Histogram\n"
+    md3 += basic_stats["model_hist_md"] + "\n"
+
+    md4 = "### Model Call (Last 24 Hours)\n"
+    md4 += basic_stats["num_chats_last_24_hours"] + "\n"
+
+    basic_component_values[0] = md0
+    basic_component_values[1] = basic_stats["chat_dates_bar"]
+    basic_component_values[2] = md1
+    basic_component_values[3] = md2
+    basic_component_values[4] = md3
+    basic_component_values[5] = md4
+
+
+def update_worker(max_num_files, interval, elo_results_file):
+    while True:
+        tic = time.time()
+        update_elo_components(max_num_files, elo_results_file)
+        durtaion = time.time() - tic
+        print(f"update duration: {durtaion:.2f} s")
+        time.sleep(max(interval - durtaion, 0))
 
 
 def load_demo(url_params, request: gr.Request):
     logger.info(f"load_demo. ip: {request.client.host}. params: {url_params}")
     return basic_component_values + leader_component_values
 
-def build_demo(elo_results_file, leaderboard_table_file):
-    from fastchat.serve.gradio_web_server import block_css
 
-    text_size = gr.themes.sizes.text_lg
-    # load theme from theme.json
-    theme = gr.themes.Default.load("theme.json")
-    # set text size to large
-    theme.text_size = text_size
-    theme.set(
-        button_large_text_size="40px",
-        button_small_text_size="40px",
-        button_large_text_weight="1000",
-        button_small_text_weight="1000",
-        button_shadow="*shadow_drop_lg",
-        button_shadow_hover="*shadow_drop_lg",
-        checkbox_label_shadow="*shadow_drop_lg",
-        button_shadow_active="*shadow_inset",
-        button_secondary_background_fill="*primary_300",
-        button_secondary_background_fill_dark="*primary_700",
-        button_secondary_background_fill_hover="*primary_200",
-        button_secondary_background_fill_hover_dark="*primary_500",
-        button_secondary_text_color="*primary_800",
-        button_secondary_text_color_dark="white",
+def model_hyperlink(model_name, link):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def load_leaderboard_table_csv(filename, add_hyperlink=True):
+    lines = open(filename).readlines()
+    heads = [v.strip() for v in lines[0].split(",")]
+    rows = []
+    for i in range(1, len(lines)):
+        row = [v.strip() for v in lines[i].split(",")]
+        for j in range(len(heads)):
+            item = {}
+            for h, v in zip(heads, row):
+                if h == "Arena Elo rating":
+                    if v != "-":
+                        v = int(ast.literal_eval(v))
+                    else:
+                        v = np.nan
+                elif h == "MMLU":
+                    if v != "-":
+                        v = round(ast.literal_eval(v) * 100, 1)
+                    else:
+                        v = np.nan
+                elif h == "MT-bench (win rate %)":
+                    if v != "-":
+                        v = round(ast.literal_eval(v[:-1]), 1)
+                    else:
+                        v = np.nan
+                elif h == "MT-bench (score)":
+                    if v != "-":
+                        v = round(ast.literal_eval(v), 2)
+                    else:
+                        v = np.nan
+                item[h] = v
+            if add_hyperlink:
+                item["Model"] = model_hyperlink(item["Model"], item["Link"])
+        rows.append(item)
+
+    return rows
+
+
+def build_basic_stats_tab():
+    empty = "Loading ..."
+    basic_component_values[:] = [empty, None, empty, empty, empty, empty]
+
+    md0 = gr.Markdown(empty)
+    gr.Markdown("#### Figure 1: Number of model calls and votes")
+    plot_1 = gr.Plot(show_label=False)
+    with gr.Row():
+        with gr.Column():
+            md1 = gr.Markdown(empty)
+        with gr.Column():
+            md2 = gr.Markdown(empty)
+    with gr.Row():
+        with gr.Column():
+            md3 = gr.Markdown(empty)
+        with gr.Column():
+            md4 = gr.Markdown(empty)
+    return [md0, plot_1, md1, md2, md3, md4]
+
+def get_full_table(arena_df, model_table_df):
+    values = []
+    for i in range(len(model_table_df)):
+        row = []
+        model_key = model_table_df.iloc[i]["key"]
+        model_name = model_table_df.iloc[i]["Model"]
+        # model display name
+        row.append(model_name)
+        if model_key in arena_df.index:
+            idx = arena_df.index.get_loc(model_key)
+            row.append(round(arena_df.iloc[idx]["rating"]))
+        else:
+            row.append(np.nan)
+        row.append(model_table_df.iloc[i]["MT-bench (score)"])
+        row.append(model_table_df.iloc[i]["MMLU"])
+        # Organization
+        row.append(model_table_df.iloc[i]["Organization"])
+        # license
+        row.append(model_table_df.iloc[i]["License"])
+
+        values.append(row)
+    values.sort(key=lambda x: -x[1] if not np.isnan(x[1]) else 1e9)
+    return values
+
+
+def get_arena_table(arena_df, model_table_df):
+    # sort by rating
+    arena_df = arena_df.sort_values(by=["rating"], ascending=False)
+    values = []
+    for i in range(len(arena_df)):
+        row = []
+        model_key = arena_df.index[i]
+        model_name = model_table_df[model_table_df["key"] == model_key]["Model"].values[
+            0
+        ]
+
+        # rank
+        ranking = arena_df.iloc[i].get("final_ranking") or i+1
+        row.append(ranking)
+        # model display name
+        row.append(model_name)
+        # elo rating
+        row.append(round(arena_df.iloc[i]["rating"]))
+        upper_diff = round(
+            arena_df.iloc[i]["rating_q975"] - arena_df.iloc[i]["rating"]
+        )
+        lower_diff = round(
+            arena_df.iloc[i]["rating"] - arena_df.iloc[i]["rating_q025"]
+        )
+        row.append(f"+{upper_diff}/-{lower_diff}")
+        # num battles
+        row.append(round(arena_df.iloc[i]["num_battles"]))
+        # Organization
+        row.append(
+            model_table_df[model_table_df["key"] == model_key]["Organization"].values[0]
+        )
+        # license
+        row.append(
+            model_table_df[model_table_df["key"] == model_key]["License"].values[0]
+        )
+
+        cutoff_date = model_table_df[model_table_df["key"] == model_key]["Knowledge cutoff date"].values[0]
+        if cutoff_date == "-":
+            row.append("Unknown")
+        else:
+            row.append(cutoff_date)
+        values.append(row)
+    return values
+
+def build_leaderboard_tab(elo_results_file, leaderboard_table_file, show_plot=False):
+    if elo_results_file is None:  # Do live update
+        default_md = "Loading ..."
+        p1 = p2 = p3 = p4 = None
+    else:
+        with open(elo_results_file, "rb") as fin:
+            elo_results = pickle.load(fin)
+            if "full" in elo_results:
+                elo_results = elo_results["full"]
+
+        p1 = elo_results["win_fraction_heatmap"]
+        p2 = elo_results["battle_count_heatmap"]
+        p3 = elo_results["bootstrap_elo_rating"]
+        p4 = elo_results["average_win_rate_bar"]
+        arena_df = elo_results["leaderboard_table_df"]
+        default_md = make_default_md(arena_df, elo_results)
+
+    md_1 = gr.Markdown(default_md, elem_id="leaderboard_markdown")
+    if leaderboard_table_file:
+        data = load_leaderboard_table_csv(leaderboard_table_file)
+        model_table_df = pd.DataFrame(data)
+
+        with gr.Tabs() as tabs:
+            # arena table
+            arena_table_vals = get_arena_table(arena_df, model_table_df)
+            with gr.Tab("Arena Elo", id=0):
+                md = make_arena_leaderboard_md(arena_df)
+                gr.Markdown(md, elem_id="leaderboard_markdown")
+                gr.Dataframe(
+                    headers=[
+                        "Rank",
+                        "🤖 Model",
+                        "⭐ Arena Elo",
+                        "📊 95% CI",
+                        "🗳️ Votes",
+                        "Organization",
+                        "License",
+                        "Knowledge Cutoff",
+                    ],
+                    datatype=[
+                        "str",
+                        "markdown",
+                        "number",
+                        "str",
+                        "number",
+                        "str",
+                        "str",
+                        "str",
+                    ],
+                    value=arena_table_vals,
+                    elem_id="arena_leaderboard_dataframe",
+                    height=700,
+                    column_widths=[50, 200, 120, 100, 100, 150, 150, 100],
+                    wrap=True,
+                )
+            with gr.Tab("Full Leaderboard", id=1):
+                md = make_full_leaderboard_md(elo_results)
+                gr.Markdown(md, elem_id="leaderboard_markdown")
+                full_table_vals = get_full_table(arena_df, model_table_df)
+                gr.Dataframe(
+                    headers=[
+                        "🤖 Model",
+                        "⭐ Arena Elo",
+                        "📈 MT-bench",
+                        "📚 MMLU",
+                        "Organization",
+                        "License",
+                    ],
+                    datatype=["markdown", "number", "number", "number", "str", "str"],
+                    value=full_table_vals,
+                    elem_id="full_leaderboard_dataframe",
+                    column_widths=[200, 100, 100, 100, 150, 150],
+                    height=700,
+                    wrap=True,
+                )
+        if not show_plot:
+            gr.Markdown(
+                """ ## Visit our [HF space](https://huggingface.co/spaces/lmsys/chatbot-arena-leaderboard) for more analysis!
+                If you want to see more models, please help us [add them](https://github.com/lm-sys/FastChat/blob/main/docs/arena.md#how-to-add-a-new-model).
+                """,
+                elem_id="leaderboard_markdown",
+            )
+    else:
+        pass
+
+    gr.Markdown(
+        f"""Note: we take the 95% confidence interval into account when determining a model's ranking.
+A model is ranked higher only if its lower bound of model score is higher than the upper bound of the other model's score.
+See Figure 3 below for visualization of the confidence intervals.
+""",
+        elem_id="leaderboard_markdown"
     )
 
+    leader_component_values[:] = [default_md, p1, p2, p3, p4]
+
+    if show_plot:
+        gr.Markdown(
+            f"""## More Statistics for Chatbot Arena\n
+Below are figures for more statistics. The code for generating them is also included in this [notebook]({notebook_url}).
+You can find more discussions in this blog [post](https://lmsys.org/blog/2023-12-07-leaderboard/).
+    """,
+            elem_id="leaderboard_markdown"
+        )
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown(
+                    "#### Figure 1: Fraction of Model A Wins for All Non-tied A vs. B Battles"
+                )
+                plot_1 = gr.Plot(p1, show_label=False)
+            with gr.Column():
+                gr.Markdown(
+                    "#### Figure 2: Battle Count for Each Combination of Models (without Ties)"
+                )
+                plot_2 = gr.Plot(p2, show_label=False)
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown(
+                    "#### Figure 3: Confidence Intervals on Model Strength (via Bootstrapping)"
+                )
+                plot_3 = gr.Plot(p3, show_label=False)
+            with gr.Column():
+                gr.Markdown(
+                    "#### Figure 4: Average Win Rate Against All Other Models (Assuming Uniform Sampling and No Ties)"
+                )
+                plot_4 = gr.Plot(p4, show_label=False)
+
+    gr.Markdown(acknowledgment_md)
+
+    if show_plot:
+        return [md_1, plot_1, plot_2, plot_3, plot_4]
+    return [md_1]
+
+block_css = """
+#notice_markdown {
+    font-size: 104%
+}
+#notice_markdown th {
+    display: none;
+}
+#notice_markdown td {
+    padding-top: 6px;
+    padding-bottom: 6px;
+}
+#leaderboard_markdown {
+    font-size: 104%
+}
+#leaderboard_markdown td {
+    padding-top: 6px;
+    padding-bottom: 6px;
+}
+#leaderboard_dataframe td {
+    line-height: 0.1em;
+}
+footer {
+    display:none !important
+}
+.sponsor-image-about img {
+    margin: 0 20px;
+    margin-top: 20px;
+    height: 40px;
+    max-height: 100%;
+    width: auto;
+    float: left;
+}
+"""
+
+acknowledgment_md = """
+### Acknowledgment
+We thank [Kaggle](https://www.kaggle.com/), [MBZUAI](https://mbzuai.ac.ae/), [a16z](https://www.a16z.com/), [Together AI](https://www.together.ai/), [Anyscale](https://www.anyscale.com/), [HuggingFace](https://huggingface.co/) for their generous [sponsorship](https://lmsys.org/donations/).
+
+<div class="sponsor-image-about">
+    <img src="https://storage.googleapis.com/public-arena-asset/kaggle.png" alt="Kaggle">
+    <img src="https://storage.googleapis.com/public-arena-asset/mbzuai.jpeg" alt="MBZUAI">
+    <img src="https://storage.googleapis.com/public-arena-asset/a16z.jpeg" alt="a16z">
+    <img src="https://storage.googleapis.com/public-arena-asset/together.png" alt="Together AI">
+    <img src="https://storage.googleapis.com/public-arena-asset/anyscale.png" alt="AnyScale">
+    <img src="https://storage.googleapis.com/public-arena-asset/huggingface.png" alt="HuggingFace">
+</div>
+"""
+
+def build_demo(elo_results_file, leaderboard_table_file):
+    text_size = gr.themes.sizes.text_lg
+
     with gr.Blocks(
         title="Chatbot Arena Leaderboard",
-        theme=theme,
+        theme=gr.themes.Base(text_size=text_size),
         css=block_css,
     ) as demo:
         leader_components = build_leaderboard_tab(
-            elo_results_file, leaderboard_table_file, arena_hard_file, show_plot=True, mirror=True
+            elo_results_file, leaderboard_table_file, show_plot=True
         )
     return demo
 
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--share", action="store_true")
-    parser.add_argument("--host", default="0.0.0.0")
-    parser.add_argument("--port", type=int, default=7860)
     args = parser.parse_args()
 
-    logger = build_logger("monitor", "monitor.log")
-    logger.info(f"args: {args}")
-
     elo_result_files = glob.glob("elo_results_*.pkl")
     elo_result_files.sort(key=lambda x: int(x[12:-4]))
     elo_result_file = elo_result_files[-1]
@@ -63,10 +453,6 @@ if __name__ == "__main__":
     leaderboard_table_files = glob.glob("leaderboard_table_*.csv")
     leaderboard_table_files.sort(key=lambda x: int(x[18:-4]))
     leaderboard_table_file = leaderboard_table_files[-1]
-    
-    arena_hard_files = glob.glob("arena_hard_auto_leaderboard_*.csv")
-    arena_hard_files.sort(key=lambda x: float(x[29:32]))
-    arena_hard_file = arena_hard_files[-1]
 
     demo = build_demo(elo_result_file, leaderboard_table_file)
-    demo.launch(share=args.share, server_name=args.host, server_port=args.port)
+    demo.launch(share=args.share)

arena_hard_auto_leaderboard_v0.1.csv

@@ -1,61 +0,0 @@
-model,score,rating_q025,rating_q975,CI,avg_tokens,date
-gpt-4-turbo-2024-04-09,82.63,80.75,84.6,"(1.9, 2.0)",662.0,2024-07-31
-claude-3-5-sonnet-20240620,79.35,77.25,80.62,"(2.1, 1.3)",567.0,2024-07-31
-gpt-4o-2024-05-13,79.21,77.42,80.71,"(1.8, 1.5)",696.0,2024-07-31
-gpt-4-0125-preview,77.96,75.94,79.9,"(2.0, 1.9)",619.0,2024-07-31
-athene-70b-0725,76.83,74.84,78.74,"(2.0, 1.9)",683.0,2024-07-31
-gpt-4o-mini-2024-07-18,74.94,72.66,77.07,"(2.3, 2.1)",668.0,2024-07-31
-gemini-1.5-pro-api-0514,71.96,69.62,74.62,"(2.3, 2.7)",676.0,2024-07-31
-yi-large-preview,71.48,69.02,73.37,"(2.5, 1.9)",720.0,2024-07-31
-mistral-large-2407,70.42,68.11,72.43,"(2.3, 2.0)",623.0,2024-07-31
-llama-3.1-405b-instruct,64.09,61.43,66.55,"(2.7, 2.5)",633.0,2024-07-31
-glm-4-0520,63.84,61.28,66.19,"(2.6, 2.3)",636.0,2024-07-31
-yi-large,63.7,61.76,65.86,"(1.9, 2.2)",626.0,2024-07-31
-deepseek-coder-v2,62.3,59.82,64.72,"(2.5, 2.4)",578.0,2024-07-31
-claude-3-opus-20240229,60.36,57.56,62.34,"(2.8, 2.0)",541.0,2024-07-31
-gemma-2-27b-it,57.51,55.11,60.12,"(2.4, 2.6)",577.0,2024-07-31
-llama-3.1-70b-instruct,55.73,52.85,58.2,"(2.9, 2.5)",628.0,2024-07-31
-glm-4-0116,55.72,53.83,58.16,"(1.9, 2.4)",622.0,2024-07-31
-gemini-1.5-pro-api-0409-preview,53.37,51.13,56.66,"(2.2, 3.3)",478.0,2024-07-31
-glm-4-air,50.88,48.62,53.21,"(2.3, 2.3)",619.0,2024-07-31
-gpt-4-0314,50.0,50.0,50.0,"(0.0, 0.0)",423.0,2024-07-31
-gemini-1.5-flash-api-0514,49.61,47.46,52.17,"(2.1, 2.6)",642.0,2024-07-31
-qwen2-72b-instruct,46.86,44.57,49.29,"(2.3, 2.4)",515.0,2024-07-31
-claude-3-sonnet-20240229,46.8,44.12,49.04,"(2.7, 2.2)",552.0,2024-07-31
-llama-3-70b-instruct,46.57,43.84,49.18,"(2.7, 2.6)",591.0,2024-07-31
-claude-3-haiku-20240307,41.47,39.57,44.02,"(1.9, 2.6)",505.0,2024-07-31
-gpt-4-0613,37.9,35.6,40.36,"(2.3, 2.5)",354.0,2024-07-31
-mistral-large-2402,37.71,34.81,39.77,"(2.9, 2.1)",400.0,2024-07-31
-mixtral-8x22b-instruct-v0.1,36.36,34.21,38.55,"(2.1, 2.2)",430.0,2024-07-31
-qwen1.5-72b-chat,36.12,33.88,38.15,"(2.2, 2.0)",474.0,2024-07-31
-phi-3-medium-4k-instruct,33.37,31.26,35.14,"(2.1, 1.8)",517.0,2024-07-31
-command-r-plus,33.07,30.85,35.12,"(2.2, 2.0)",541.0,2024-07-31
-mistral-medium,31.9,29.66,34.31,"(2.2, 2.4)",485.0,2024-07-31
-phi-3-small-8k-instruct,29.77,27.94,31.97,"(1.8, 2.2)",568.0,2024-07-31
-mistral-next,27.37,25.4,29.09,"(2.0, 1.7)",297.0,2024-07-31
-gpt-3.5-turbo-0613,24.82,22.54,26.29,"(2.3, 1.5)",401.0,2024-07-31
-dbrx-instruct-preview,24.63,22.33,26.83,"(2.3, 2.2)",415.0,2024-07-31
-claude-2.0,23.99,21.71,25.65,"(2.3, 1.7)",295.0,2024-07-31
-mixtral-8x7b-instruct-v0.1,23.4,21.38,25.41,"(2.0, 2.0)",457.0,2024-07-31
-gpt-3.5-turbo-0125,23.34,21.67,25.27,"(1.7, 1.9)",329.0,2024-07-31
-yi-34b-chat,23.15,20.75,24.7,"(2.4, 1.6)",611.0,2024-07-31
-starling-lm-7b-beta,23.01,20.81,24.66,"(2.2, 1.6)",530.0,2024-07-31
-claude-2.1,22.77,20.65,25.43,"(2.1, 2.7)",290.0,2024-07-31
-llama-3.1-8b-instruct,21.34,19.71,23.09,"(1.6, 1.8)",861.0,2024-07-31
-snorkel-mistral-pairrm-dpo,20.73,19.04,22.05,"(1.7, 1.3)",564.0,2024-07-31
-llama-3-8b-instruct,20.56,18.82,22.61,"(1.7, 2.1)",585.0,2024-07-31
-gpt-3.5-turbo-1106,18.87,17.06,20.58,"(1.8, 1.7)",285.0,2024-07-31
-gpt-3.5-turbo-0314,18.05,16.57,20.06,"(1.5, 2.0)",334.0,2024-07-31
-gemini-pro,17.8,15.96,19.32,"(1.8, 1.5)",322.0,2024-07-31
-snowflake-arctic-instruct,17.61,16.12,19.27,"(1.5, 1.7)",365.0,2024-07-31
-command-r,17.02,15.73,18.51,"(1.3, 1.5)",432.0,2024-07-31
-phi-3-mini-128k-instruct,15.43,13.94,17.02,"(1.5, 1.6)",609.0,2024-07-31
-tulu-2-dpo-70b,14.99,13.05,16.82,"(1.9, 1.8)",550.0,2024-07-31
-starling-lm-7b-alpha,12.8,11.23,14.5,"(1.6, 1.7)",483.0,2024-07-31
-mistral-7b-instruct,12.57,11.05,14.11,"(1.5, 1.5)",541.0,2024-07-31
-gemma-1.1-7b-it,12.09,10.61,13.43,"(1.5, 1.3)",341.0,2024-07-31
-llama-2-70b-chat,11.55,10.02,13.01,"(1.5, 1.5)",595.0,2024-07-31
-vicuna-33b,8.63,7.59,9.84,"(1.0, 1.2)",451.0,2024-07-31
-gemma-7b-it,7.47,6.5,8.6,"(1.0, 1.1)",378.0,2024-07-31
-gemma-1.1-2b-it,3.37,2.74,4.14,"(0.6, 0.8)",316.0,2024-07-31
-gemma-2b-it,3.0,2.33,3.67,"(0.7, 0.7)",369.0,2024-07-31

elo_results_20240327.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:bab4e9fa00e9d7c8244723993174af2c4f35ffc8487cc3059504b72658f06f43
-size 457743

elo_results_20240410.pkl → elo_results_20240329.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8a5757ab9692c6121451f2c787700507fe6b866837329ab0a47a9003a274338f
-size 120963
+oid sha256:7f4c037f68c9ddbf27b70b1cb333ca37bf70ff9a3cddad7a93cd62bca709cd77
+size 115776

elo_results_20240403.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ce8cebf41da8c06eee0f37156e01be83cc43182e0f00444311b4ad97a83154be
-size 690286

elo_results_20240409.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e6774f780b63f569666e9a85b12eddceef3af75e1d1799ff7c6e0529102950c3
-size 119947

elo_results_20240411.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:fada8d86ddb6dae319c5bda602d921859cc4280fdd53388eff446d80c3ab8192
-size 1183214

elo_results_20240413.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3ad8ebb2a8602a7c72382fc65521fbe7b06bb36dcf6b6cc582c6b89b1d7b1a87
-size 1064654

elo_results_20240418.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1b05163db100df9ef62c7efb3332891d3321c6094e787af3b4ef4a9afe2becdb
-size 1130887

elo_results_20240419.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:05239b0b4406f270fdc845632981024dec639b8351dcb1a2308def3bbcea2e68
-size 1130756

elo_results_20240422.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:436c6bc88b6c03b672a3a87ddb3b101ec1c5ff03d47d64196986b5d6ca7909cd
-size 1254718

elo_results_20240426.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a39b33094ac93d4a1e0bc57bfbb17368515ce5a7e4504d3d1e310a14cd056943
-size 1275849

elo_results_20240501.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:35f506d5d34555cbc055abc901623fae3aa7b429057cf3039cb1b460fdc8f41c
-size 1159628

elo_results_20240508.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:79acc98f859a5f597338eff0ae98025abfec80087d60336c0d735e7dd3595eb2
-size 1188396

elo_results_20240515.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:692e9b280f1587b85f28aaeceee52928a92f6a98ee81e3d63a2d789c82eb9abc
-size 1596977

elo_results_20240516.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e3ff3e278c66aa28aece3e52369d128ca6707673e0e811304294c5d8a85aaf81
-size 1476125

elo_results_20240519.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:70c1136517de9396d72b2e14faee88382e503d0b80e56a5131e220173f6b472b
-size 1604729

elo_results_20240520.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:82d5306e5b88813cea3490a0cf2d02952219c52257789d6077caeee986996567
-size 1628933

elo_results_20240527.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ca33433b15176a4b7c4f13584b2129ebd60f059524648c4a4a986aad4a84bc1e
-size 1666346

elo_results_20240602.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1e884bc9a41a3aa9916e29ca34e5ac2e52e6f8d7e314e380facd06f9ae855145
-size 2278603

elo_results_20240606.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:6073ca1b10e9ac34c2b67b73636dcd2303b4a9291c9a440ad3813c33ef5fa170
-size 2295194

elo_results_20240611.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8b0f14fede0abeee60825682ffab3b07b50af5d9924de3c8114ddac469b34779
-size 2310921

elo_results_20240617.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a6e53f4d339bbc3420b349e89a315a1c6d2fa3c9847b206aeb02e1e5170aea73
-size 2491948

elo_results_20240621.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b098f9f69009376d87ef4d317c81c648d0558f7c912f693e285de5d3115e309b
-size 2526260

elo_results_20240623.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1469aac4504cd49bdbf1566093c5be912fd6ffe27f62213fd4961eefc92b4e30
-size 2544361

elo_results_20240626.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f6b7c6dc746e8dfb7fed966d9c027468dccbd31da5866af6fff0083478ef52ff
-size 2429037

elo_results_20240629.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:acdaa9f0e89e01a5d1ee914d750727bf9877cde8bf2e195439bc7625b80f197f
-size 2679801

elo_results_20240706.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:dbd774b3b25712428c96cf54b5cb8c4d912e8b8215edbcee3dee0974fd898c8e
-size 2702290

elo_results_20240708.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:5d72cd3bf16c9af40910a2faf9f1403df2788596390e0f13452835748968679d
-size 3154928

elo_results_20240716.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:dec0c95bbaf8a2ed382b1676eba0b22ad89fa8815d28bc22c94cc6ea00205e5d
-size 3029902

elo_results_20240722.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c01b5a786ffd8bf64b72d39ca3fc7dee5483852c6a515f515532c8096a6e16d4
-size 3054518

elo_results_20240725.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1bfde449e424f12c9316f0c64062fc1ef9926e4a924bbaeafa455fcde0decb6f
-size 3073542

elo_results_20240730.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8db59f5f0852f26600f85c3188547e5d97b4641906c979bcc178cd5be7a7554c
-size 3137995

elo_results_20240731.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9aaff49a611fef8ff7bec058827d7de3426973c15bc538ac6bb39e764cc14b34
-size 3157482

elo_results_20240801.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:cdaa16f762adc54cbe05ab03dff88587491193852846b749980294b7f1ea2bec
-size 3182414

elo_results_20240805.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e638bfe870e4409c4d768d57287cc8f6310caf85191d8020ba89f7f59ee9f6d8
-size 3202114

elo_results_20240806.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a4f31f36636a280589bb1039f9d1f405a989df6a8f74d1af30555d891b23a416
-size 3261205

elo_results_20240813.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b4c5dfd4247b704f07e61ae27dc0642a3d3cfa9a6872cc3dc03d1888a594de9f
-size 2943734

elo_results_20240822.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c733894de1252232a63ce3632abe52504fb6bcf43e17bb49fea2b5ad8d76116f
-size 3004697

elo_results_20240823.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ec103aa6bf5d0f02f8bd2c69c8ccfc8f1be1b44c7dc004d967c8d5ce470975b5
-size 3039588

elo_results_20240827.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:fbcf63bc492b9e2018fdd2c82924f375f12db20dd577f2c139a8ff82a2d08159
-size 3093445

elo_results_20240828.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b7da13b5f061a7a5a112e5ca45ff707d6cf6259c8a01b40ea5b77bbd5bd3d5b0
-size 3819732

elo_results_20240904.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:cf6117b1e28bb982e965d20b927685dce98750a82c255b868588b5b2318aaee9
-size 3486555

elo_results_20240915.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:42512fe148c81eeab05961ed64fe446dc7f6ed3703f976fc4c8c2a6a3a3e6bef
-size 3726145

elo_results_20240917.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d15ed36a6e429a9213e9230fbd28de05e7788f758c4282660dd77a4689f98590
-size 3768775

elo_results_20240927.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7c96f9a95fbbf44b8cb46129cc8ea09eef1bb6e43b8ec12d1c8837091d23ee69
-size 3860967

elo_results_20241007.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3891408ad17e52226989783562eb75f784dfb1d0a30dd9051e943b48cd0b117c
-size 3919717

elo_results_20241015.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:6daea1d6b24298f32f23c903872aa6c852453de29bf82aa0d2e7aaf5337af715
-size 4058396

elo_results_20241023.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2e592a6807b4affbc2e203b385ff491743116b4854c9bcd3c3bbf037fba6092f
-size 4374135