File size: 5,907 Bytes
3ebc784 5b15f5e 3ebc784 7eda93e ec727b9 3ebc784 5b15f5e 3ebc784 5b15f5e 3ebc784 5b15f5e 3ebc784 5b15f5e 3ebc784 5b15f5e ec727b9 5b15f5e ec727b9 5b15f5e ec727b9 5b15f5e ec727b9 6df95f1 5b15f5e 3ebc784 13ff3a0 a2b0b51 b9d3833 2f02c91 4a498d2 13ff3a0 4afa91d f55aa6a e056b5e 3ebc784 7eda93e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 |
# source: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/blob/main/src/utils_display.py
from dataclasses import dataclass
import plotly.graph_objects as go
from transformers import AutoConfig
# These classes are for user facing column names, to avoid having to change them
# all around the code when a modif is needed
@dataclass
class ColumnContent:
name: str
type: str
displayed_by_default: bool
hidden: bool = False
def fields(raw_class):
return [
v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"
]
@dataclass(frozen=True)
class AutoEvalColumn: # Auto evals column
model_type_symbol = ColumnContent("T", "str", True)
model = ColumnContent("Models", "markdown", True)
win_rate = ColumnContent("Win Rate", "number", True)
average = ColumnContent("Average score", "number", False)
humaneval_python = ColumnContent("humaneval-python", "number", True)
java = ColumnContent("java", "number", True)
javascript = ColumnContent("javascript", "number", True)
throughput = ColumnContent("Throughput (tokens/s)", "number", True)
cpp = ColumnContent("cpp", "number", False)
php = ColumnContent("php", "number", False)
rust = ColumnContent("rust", "number", False)
swift = ColumnContent("swift", "number", False)
r = ColumnContent("r", "number", False)
lua = ColumnContent("lua", "number", False)
d = ColumnContent("d", "number", False)
racket = ColumnContent("racket", "number", False)
julia = ColumnContent("julia", "number", False)
languages = ColumnContent("#Languages", "number", False)
throughput_bs50 = ColumnContent("Throughput (tokens/s) bs=50", "number", False)
peak_memory = ColumnContent("Peak Memory (MB)", "number", False)
seq_length = ColumnContent("Seq_length", "number", False)
link = ColumnContent("Links", "str", False)
dummy = ColumnContent("Models", "str", True)
pr = ColumnContent("Submission PR", "str", False)
def model_hyperlink(link, model_name):
return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
def make_clickable_names(df):
df["Models"] = df.apply(
lambda row: model_hyperlink(row["Links"], row["Models"]), axis=1
)
return df
def plot_throughput(df, bs=1):
throughput_column = (
"Throughput (tokens/s)" if bs == 1 else "Throughput (tokens/s) bs=50"
)
df["symbol"] = 2 # Triangle
df["color"] = ""
df.loc[df["Models"].str.contains("StarCoder|SantaCoder"), "color"] = "orange"
df.loc[df["Models"].str.contains("CodeGen"), "color"] = "pink"
df.loc[df["Models"].str.contains("Replit"), "color"] = "purple"
df.loc[df["Models"].str.contains("WizardCoder"), "color"] = "peru"
df.loc[df["Models"].str.contains("CodeGeex"), "color"] = "cornflowerblue"
df.loc[df["Models"].str.contains("StableCode"), "color"] = "cadetblue"
df.loc[df["Models"].str.contains("OctoCoder"), "color"] = "lime"
df.loc[df["Models"].str.contains("OctoGeeX"), "color"] = "wheat"
df.loc[df["Models"].str.contains("Deci"), "color"] = "salmon"
df.loc[df["Models"].str.contains("CodeLlama"), "color"] = "palevioletred"
df.loc[df["Models"].str.contains("CodeGuru"), "color"] = "burlywood"
df.loc[df["Models"].str.contains("Phind"), "color"] = "crimson"
df.loc[df["Models"].str.contains("Falcon"), "color"] = "dimgray"
df.loc[df["Models"].str.contains("Refact"), "color"] = "yellow"
df.loc[df["Models"].str.contains("Phi"), "color"] = "gray"
fig = go.Figure()
for i in df.index:
fig.add_trace(
go.Scatter(
x=[df.loc[i, throughput_column]],
y=[df.loc[i, "Average score"]],
mode="markers",
marker=dict(
size=[df.loc[i, "Size (B)"] + 10],
color=df.loc[i, "color"],
symbol=df.loc[i, "symbol"],
),
name=df.loc[i, "Models"],
hovertemplate="<b>%{text}</b><br><br>"
+ f"{throughput_column}: %{{x}}<br>"
+ "Average Score: %{y}<br>"
+ "Peak Memory (MB): "
+ str(df.loc[i, "Peak Memory (MB)"])
+ "<br>"
+ "Human Eval (Python): "
+ str(df.loc[i, "humaneval-python"]),
text=[df.loc[i, "Models"]],
showlegend=True,
)
)
fig.update_layout(
autosize=False,
width=650,
height=600,
title=f"Average Score Vs Throughput (A100-80GB, Float16, Batch Size <b>{bs}</b>)",
xaxis_title=f"{throughput_column}",
yaxis_title="Average Code Score",
)
return fig
def styled_error(error):
return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
def styled_warning(warn):
return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
def styled_message(message):
return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
def has_no_nan_values(df, columns):
return df[columns].notna().all(axis=1)
def has_nan_values(df, columns):
return df[columns].isna().any(axis=1)
def is_model_on_hub(model_name: str, revision: str) -> bool:
try:
AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=False)
return True, None
except ValueError:
return (
False,
"needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
)
except Exception as e:
print(f"Could not get the model config from the hub.: {e}")
return False, "was not found on hub!" |