File size: 1,416 Bytes
d5c587b
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
Models,Size (B),Average score,Throughput (tokens/s),Throughput (tokens/s) bs=50,Seq_length,#Languages,humaneval-python,java,javascript,cpp,php,julia,d,lua,r,ruby,racket,rust,swift,Peak Memory (MB)
StarCoder-15B,15.0,22.83,38.6,1490.0,8192,86,33.57,30.22,30.79,31.55,26.08,23.02,13.57,23.89,15.5,1.24,0.07,21.84,22.74,33461
StarCoderBase-15B,15.0,22.0,44.2,1460.0,8192,86,30.35,28.53,31.7,30.56,26.75,21.09,10.01,26.61,10.18,17.25,11.77,24.46,16.74,32366
StarCoderBase-7B,7.0,20.04,43.1,1700.0,8192,86,28.37,24.44,27.35,23.3,22.12,21.77,8.1,23.35,14.51,18.39,11.08,22.6,15.1,16512
CodeGen25-7B-multi,7.0,19.86,32.6,680.0,2048,86,28.7,26.01,26.27,25.75,21.98,19.11,8.84,23.44,11.59,17.72,10.37,21.84,16.62,15336
StarCoderBase-3B,3.0,14.41,50.0,1770.0,8192,86,21.5,19.25,21.32,19.43,18.55,16.1,4.97,18.04,10.1,3.93,7.87,16.32,9.98,8414
CodeGen25-7B-mono,7.0,12.1,34.1,687.0,2048,86,33.08,19.75,23.22,18.62,16.75,4.65,4.32,6.75,4.41,0.0,4.07,7.83,1.71,15336
CodeGen-16B-Multi,16.0,11.81,17.2,0.0,2048,6,19.26,22.2,19.15,21.0,8.37,0.0,7.68,8.5,6.45,0.0,0.66,4.21,1.25,32890
Replit-2.7B,2.7,11.55,42.2,577.0,2048,20,20.12,21.39,20.18,20.37,16.14,1.24,6.41,2.11,7.2,10.75,3.22,15.19,5.88,7176
StarCoderBase-1.1B,1.1,9.81,71.4,2360.0,8192,86,15.17,14.2,13.38,11.68,9.94,11.31,4.65,12.52,5.73,0.31,5.03,10.24,3.92,4586
SantaCoder-1.1B,1.1,9.72,50.8,2270.0,2048,3,18.12,15.0,15.47,6.2,1.5,0.0,0.0,0.1,0.0,0.0,0.0,2.0,0.7,4602