loubnabnl HF staff commited on
Commit
80a6d24
1 Parent(s): 2b8f53a

add models

Browse files
Files changed (2) hide show
  1. code_eval_board.csv +6 -6
  2. src/build.py +1 -6
code_eval_board.csv CHANGED
@@ -1,13 +1,13 @@
1
  Models,Size (B),Win Rate,Average score,Throughput (tokens/s),Seq_length,#Languages,humaneval-python,java,javascript,cpp,php,julia,d,lua,r,ruby,racket,rust,swift,Throughput (tokens/s) bs=50,Peak Memory (MB)
2
  WizardCoder-15B-V1.0,15.0,11.54,29.26,43.7,8192,86,50.53,35.77,41.91,38.95,39.34,33.98,12.14,27.85,22.53,3.21,13.39,33.74,27.06,1470.0,32414.0
3
- StarCoder-15B,15.0,9.58,22.83,43.9,8192,86,33.57,30.22,30.79,31.55,26.08,23.02,13.57,23.89,15.5,1.24,0.07,21.84,22.74,1490.0,33461.0
4
  StarCoderBase-15B,15.0,9.85,22.0,43.8,8192,86,30.35,28.53,31.7,30.56,26.75,21.09,10.01,26.61,10.18,17.25,11.77,24.46,16.74,1460.0,32366.0
 
5
  StarCoderBase-7B,7.0,8.46,20.04,46.9,8192,86,28.37,24.44,27.35,23.3,22.12,21.77,8.1,23.35,14.51,18.39,11.08,22.6,15.1,1700.0,16512.0
6
  CodeGen25-7B-multi,7.0,8.35,19.86,32.6,2048,86,28.7,26.01,26.27,25.75,21.98,19.11,8.84,23.44,11.59,17.72,10.37,21.84,16.62,680.0,15336.0
7
- CodeGeex2-6B,6.0,7.46,18.7,32.7,8192,23,33.49,22.79,12.57,26.37,24.07,15.64,8.14,14.16,15.11,0.01,11.35,19.94,20.81,,
8
  StarCoderBase-3B,3.0,5.77,14.41,50.0,8192,86,21.5,19.25,21.32,19.43,18.55,16.1,4.97,18.04,10.1,3.93,7.87,16.32,9.98,1770.0,8414.0
9
- CodeGen25-7B-mono,7.0,4.0,12.1,34.1,2048,86,33.08,19.75,23.22,18.62,16.75,4.65,4.32,6.75,4.41,0.0,4.07,7.83,1.71,687.0,15336.0
10
- CodeGen-16B-Multi,16.0,3.5,11.81,17.2,2048,6,19.26,22.2,19.15,21.0,8.37,0.0,7.68,8.5,6.45,0.0,0.66,4.21,1.25,0.0,32890.0
11
  Replit-2.7B,2.7,4.69,11.55,42.2,2048,20,20.12,21.39,20.18,20.37,16.14,1.24,6.41,2.11,7.2,10.75,3.22,15.19,5.88,577.0,7176.0
12
- StarCoderBase-1.1B,1.1,3.38,9.81,71.4,8192,86,15.17,14.2,13.38,11.68,9.94,11.31,4.65,12.52,5.73,0.31,5.03,10.24,3.92,2360.0,4586.0
13
- SantaCoder-1.1B,1.1,1.42,9.72,50.8,2048,3,18.12,15.0,15.47,6.2,1.5,0.0,0.0,0.1,0.0,0.0,0.0,2.0,0.7,2270.0,4602.0
 
 
 
1
  Models,Size (B),Win Rate,Average score,Throughput (tokens/s),Seq_length,#Languages,humaneval-python,java,javascript,cpp,php,julia,d,lua,r,ruby,racket,rust,swift,Throughput (tokens/s) bs=50,Peak Memory (MB)
2
  WizardCoder-15B-V1.0,15.0,11.54,29.26,43.7,8192,86,50.53,35.77,41.91,38.95,39.34,33.98,12.14,27.85,22.53,3.21,13.39,33.74,27.06,1470.0,32414.0
 
3
  StarCoderBase-15B,15.0,9.85,22.0,43.8,8192,86,30.35,28.53,31.7,30.56,26.75,21.09,10.01,26.61,10.18,17.25,11.77,24.46,16.74,1460.0,32366.0
4
+ StarCoder-15B,15.0,9.58,21.08,43.9,8192,86,33.57,30.22,30.79,31.55,26.08,23.02,13.57,23.89,15.5,1.24,0.07,21.84,22.74,1490.0,33461.0
5
  StarCoderBase-7B,7.0,8.46,20.04,46.9,8192,86,28.37,24.44,27.35,23.3,22.12,21.77,8.1,23.35,14.51,18.39,11.08,22.6,15.1,1700.0,16512.0
6
  CodeGen25-7B-multi,7.0,8.35,19.86,32.6,2048,86,28.7,26.01,26.27,25.75,21.98,19.11,8.84,23.44,11.59,17.72,10.37,21.84,16.62,680.0,15336.0
7
+ CodeGeex2-6B,6.0,7.46,17.27,32.7,8192,23,33.49,22.79,12.57,26.37,24.07,15.64,8.14,14.16,15.11,0.01,11.35,19.94,20.81,,
8
  StarCoderBase-3B,3.0,5.77,14.41,50.0,8192,86,21.5,19.25,21.32,19.43,18.55,16.1,4.97,18.04,10.1,3.93,7.87,16.32,9.98,1770.0,8414.0
 
 
9
  Replit-2.7B,2.7,4.69,11.55,42.2,2048,20,20.12,21.39,20.18,20.37,16.14,1.24,6.41,2.11,7.2,10.75,3.22,15.19,5.88,577.0,7176.0
10
+ CodeGen25-7B-mono,7.0,4.0,11.17,34.1,2048,86,33.08,19.75,23.22,18.62,16.75,4.65,4.32,6.75,4.41,0.0,4.07,7.83,1.71,687.0,15336.0
11
+ CodeGen-16B-Multi,16.0,3.5,9.13,17.2,2048,6,19.26,22.2,19.15,21.0,8.37,0.0,7.68,8.5,6.45,0.0,0.66,4.21,1.25,0.0,32890.0
12
+ StarCoderBase-1.1B,1.1,3.38,9.08,71.4,8192,86,15.17,14.2,13.38,11.68,9.94,11.31,4.65,12.52,5.73,0.31,5.03,10.24,3.92,2360.0,4586.0
13
+ SantaCoder-1.1B,1.1,1.42,4.55,50.8,2048,3,18.12,15.0,15.47,6.2,1.5,0.0,0.0,0.1,0.0,0.0,0.0,2.0,0.7,2270.0,4602.0
src/build.py CHANGED
@@ -42,12 +42,7 @@ data = {
42
  df = pd.DataFrame(data).set_index("Models")
43
  df = df.reset_index().rename(columns={"index": "Language"})
44
 
45
- temp_df = df.copy()
46
- temp_df = temp_df.apply(pd.to_numeric, errors="coerce")
47
- temp_df[temp_df <= 1] = np.nan
48
- temp_ = temp_df.iloc[:, 5:-2]
49
- print(temp_)
50
- df.insert(2, "Average score", temp_.mean(axis=1).round(2))
51
 
52
  # add win rate columns for each language
53
  old_size = len(df.columns)
 
42
  df = pd.DataFrame(data).set_index("Models")
43
  df = df.reset_index().rename(columns={"index": "Language"})
44
 
45
+ df.insert(2, "Average score", df.iloc[:, 5:-2].mean(axis=1).round(2))
 
 
 
 
 
46
 
47
  # add win rate columns for each language
48
  old_size = len(df.columns)