fix average
Browse files- data/raw_scores.csv +13 -0
- src/build.py +2 -2
data/raw_scores.csv
ADDED
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
Models,Size (B),Throughput (tokens/s),Seq_length,#Languages,humaneval-python,java,javascript,cpp,php,julia,d,lua,r,racket,rust,swift,Throughput (tokens/s) bs=50,Peak Memory (MB)
|
2 |
+
CodeGen-16B-Multi,16.0,17.2,2048,6,19.26,22.2,19.15,21.0,8.37,0.0,7.68,8.5,6.45,0.66,4.21,1.25,0.0,32890
|
3 |
+
StarCoder-15B,15.0,43.9,8192,86,33.57,30.22,30.79,31.55,26.08,23.02,13.57,23.89,15.5,0.07,21.84,22.74,1490.0,33461
|
4 |
+
StarCoderBase-15B,15.0,43.8,8192,86,30.35,28.53,31.7,30.56,26.75,21.09,10.01,26.61,10.18,11.77,24.46,16.74,1460.0,32366
|
5 |
+
StarCoderBase-7B,7.0,46.9,8192,86,28.37,24.44,27.35,23.3,22.12,21.77,8.1,23.35,14.51,11.08,22.6,15.1,1700.0,16512
|
6 |
+
StarCoderBase-3B,3.0,50.0,8192,86,21.5,19.25,21.32,19.43,18.55,16.1,4.97,18.04,10.1,7.87,16.32,9.98,1770.0,8414
|
7 |
+
Replit-2.7B,2.7,42.2,2048,20,20.12,21.39,20.18,20.37,16.14,1.24,6.41,2.11,7.2,3.22,15.19,5.88,577.0,7176
|
8 |
+
SantaCoder-1.1B,1.1,50.8,2048,3,18.12,15.0,15.47,6.2,1.5,0.0,0.0,0.1,0.0,0.0,2.0,0.7,2270.0,4602
|
9 |
+
StarCoderBase-1.1B,1.1,71.4,8192,86,15.17,14.2,13.38,11.68,9.94,11.31,4.65,12.52,5.73,5.03,10.24,3.92,2360.0,4586
|
10 |
+
CodeGen25-7B-mono,7.0,34.1,2048,86,33.08,19.75,23.22,18.62,16.75,4.65,4.32,6.75,4.41,4.07,7.83,1.71,687.0,15336
|
11 |
+
CodeGen25-7B-multi,7.0,32.6,2048,86,28.7,26.01,26.27,25.75,21.98,19.11,8.84,23.44,11.59,10.37,21.84,16.62,680.0,15336
|
12 |
+
CodeGeex2-6B,6.0,32.7,8192,100,34.54,23.46,29.9,28.45,25.27,20.93,8.44,15.94,14.58,11.75,20.45,22.06,1100,14110
|
13 |
+
WizardCoder-15B-V1.0,15.0,43.7,8192,86,50.53,35.77,41.91,38.95,39.34,33.98,12.14,27.85,22.53,13.39,33.74,27.06,1470.0,32414
|
src/build.py
CHANGED
@@ -5,8 +5,9 @@ df = pd.read_csv("data/raw_scores.csv")
|
|
5 |
COLS = df.columns.to_list()
|
6 |
# add column models_query with same values a smodels at the end of columns
|
7 |
df.insert(len(COLS), "models_query", df["Models"])
|
|
|
8 |
# average score
|
9 |
-
df.insert(2, "Average score", df.iloc[:, 5:-
|
10 |
|
11 |
# add win rate columns for each language
|
12 |
old_size = len(df.columns)
|
@@ -43,4 +44,3 @@ df.insert(0, "T", "🟢")
|
|
43 |
df.loc[df["Models"].str.contains("WizardCoder"), "T"] = "🔶"
|
44 |
print(df)
|
45 |
df.to_csv("data/code_eval_board.csv", index=False)
|
46 |
-
# print first 10 cols
|
|
|
5 |
COLS = df.columns.to_list()
|
6 |
# add column models_query with same values a smodels at the end of columns
|
7 |
df.insert(len(COLS), "models_query", df["Models"])
|
8 |
+
|
9 |
# average score
|
10 |
+
df.insert(2, "Average score", df.iloc[:, 5:-3].mean(axis=1).round(2))
|
11 |
|
12 |
# add win rate columns for each language
|
13 |
old_size = len(df.columns)
|
|
|
44 |
df.loc[df["Models"].str.contains("WizardCoder"), "T"] = "🔶"
|
45 |
print(df)
|
46 |
df.to_csv("data/code_eval_board.csv", index=False)
|
|