File size: 12,331 Bytes
c47d747 d242132 c99e6b7 d242132 c99e6b7 d242132 c99e6b7 d242132 c99e6b7 d242132 c99e6b7 d242132 c99e6b7 d242132 c99e6b7 d242132 c99e6b7 d242132 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 |
T,Model,Size (B),Win Rate,Throughput (tokens/s),Seq_length,#Languages,humaneval-python,java,javascript,cpp,php,julia,d,Average score,lua,r,racket,rust,swift,Throughput (tokens/s) bs=50,Peak Memory (MB),models_query,Links,Submission PR
🔶 EXT,OpenCodeInterpreter-DS-33B,33.0,54.67,,16384,86,75.23,54.8,69.06,64.47,59.32,46.58,22.31,53.32,57.76,45.94,34.55,57.95,51.85,,,OpenCodeInterpreter-DS-33B,https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-33B,https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard/discussions/60
🔶,CodeQwen1.5-7B-Chat,7.0,53.83,,65536,92,87.2,61.04,70.31,67.85,69.22,51.89,22.71,54.95,32.89,39.86,29.38,67.49,59.59,,,CodeQwen1.5-7B-Chat,https://huggingface.co/Qwen/CodeQwen1.5-7B-Chat,
🔶 EXT,CodeFuse-DeepSeek-33b,33.0,53.08,17.5,16384,86,76.83,60.76,66.46,65.22,57.76,38.36,24.36,51.69,52.8,40.37,34.16,53.85,49.37,,75833.0,CodeFuse-DeepSeek-33b,https://huggingface.co/codefuse-ai/CodeFuse-DeepSeek-33B,https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard/discussions/51
🔶 EXT,DeepSeek-Coder-33b-instruct,33.0,51.33,25.2,16384,86,80.02,52.03,65.13,62.36,52.5,42.92,17.85,49.99,50.92,39.43,31.69,55.56,49.42,,76800.0,DeepSeek-Coder-33b-instruct,https://huggingface.co/deepseek-ai/deepseek-coder-33b-instruct,https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard/discussions/42
🔶 EXT,DeepSeek-Coder-7b-instruct,6.7,49.92,51.0,16384,86,80.22,53.34,65.8,59.66,59.4,38.84,21.59,48.17,47.78,38.56,20.87,47.73,44.22,,22922.0,DeepSeek-Coder-7b-instruct,https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct,https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard/discussions/43
🔶 EXT,OpenCodeInterpreter-DS-6.7B,6.7,49.33,,16384,86,73.2,51.41,63.85,60.01,57.34,39.69,18.22,47.14,44.3,39.08,24.32,48.22,45.99,,,OpenCodeInterpreter-DS-6.7B,https://huggingface.co/m-a-p/OpenCodeInterpreter-DS-6.7B,https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard/discussions/61
🔶,Phind-CodeLlama-34B-v2,34.0,48.5,15.1,16384,UNK,71.95,54.06,65.34,59.59,56.26,45.12,14.12,48.7,44.27,37.7,28.7,57.67,49.63,0.0,69957.0,Phind-CodeLlama-34B-v2,https://huggingface.co/phind/Phind-CodeLlama-34B-v2,
🔶,Phind-CodeLlama-34B-v1,34.0,47.38,15.1,16384,UNK,65.85,49.47,64.45,57.81,55.53,43.23,15.5,46.9,42.05,36.71,24.89,54.1,53.27,0.0,69957.0,Phind-CodeLlama-34B-v1,https://huggingface.co/phind/Phind-CodeLlama-34B-v1,
🔶,Phind-CodeLlama-34B-Python-v1,34.0,45.81,15.1,16384,UNK,70.22,48.72,66.24,55.34,52.05,44.23,13.78,45.25,39.44,37.76,18.88,49.22,47.11,0.0,69957.0,Phind-CodeLlama-34B-Python-v1,https://huggingface.co/phind/Phind-CodeLlama-34B-Python-v1,
🟢,CodeQwen1.5-7B,7.0,44.92,,65536,92,50.79,42.15,50.07,48.35,50.92,37.27,16.03,40.47,46.29,27.95,24.89,47.56,43.33,,,CodeQwen1.5-7B,https://huggingface.co/Qwen/CodeQwen1.5-7B,
🔶,CodeLlama-70b-Instruct,70.0,43.33,,2048,UNK,75.6,47.2,57.76,48.45,57.14,42.24,19.88,42.64,44.1,29.19,0.0,47.2,42.86,,,CodeLlama-70b-Instruct,https://huggingface.co/codellama/CodeLlama-70b-Instruct-hf,
🔶,WizardCoder-Python-34B-V1.0,34.0,43.04,15.1,16384,UNK,70.73,44.94,55.28,47.2,47.2,41.51,15.38,41.95,32.3,39.75,18.63,46.15,44.3,0.0,69957.0,WizardCoder-Python-34B-V1.0,https://huggingface.co/WizardLM/WizardCoder-Python-34B-V1.0,
🟢,CodeLlama-70b,70.0,42.79,,16384,UNK,52.44,44.72,56.52,49.69,46.58,42.24,24.84,39.93,41.61,27.95,0.0,49.69,42.86,,,CodeLlama-70b,https://huggingface.co/codellama/CodeLlama-70b-hf,
🟢 EXT,DeepSeek-Coder-33b-base,33.0,42.75,25.2,16384,86,52.45,43.77,51.28,51.22,41.76,32.83,17.41,38.07,36.51,26.76,23.37,43.78,35.75,,76800.0,DeepSeek-Coder-33b-base,https://huggingface.co/deepseek-ai/deepseek-coder-33b-base,https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard/discussions/31
🟢,CodeLlama-70b-Python,70.0,42.08,,2048,UNK,55.49,45.96,56.52,49.69,52.8,35.4,21.12,39.61,44.72,26.09,0.0,48.45,39.13,,,CodeLlama-70b-Python,https://huggingface.co/codellama/CodeLlama-70b-Python-hf,
🟢,StarCoder2-15B,15.0,39.67,,16384,619,44.15,33.86,44.24,41.44,39.48,33.19,23.64,34.85,43.75,19.81,22.41,38.03,34.18,,,StarCoder2-15B,https://huggingface.co/bigcode/starcoder2-15b,
🟢 EXT,DeepSeek-Coder-7b-base,6.7,38.33,51.0,16384,86,45.83,37.72,45.9,45.53,36.92,28.74,19.74,33.54,33.89,28.99,18.73,34.67,25.8,,22922.0,DeepSeek-Coder-7b-base,https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base,https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard/discussions/32
🔶,CodeLlama-34b-Instruct,34.0,38.27,15.1,16384,UNK,50.79,41.53,45.85,41.53,36.98,32.65,13.63,35.09,38.87,24.25,18.09,39.26,37.63,0.0,69957.0,CodeLlama-34b-Instruct,https://huggingface.co/codellama/CodeLlama-34b-Instruct-hf,
🔶,WizardCoder-Python-13B-V1.0,13.0,38.04,25.3,16384,UNK,62.19,41.77,48.45,42.86,42.24,38.99,11.54,35.94,32.92,27.33,16.15,34.62,32.28,0.0,28568.0,WizardCoder-Python-13B-V1.0,https://huggingface.co/WizardLM/WizardCoder-Python-13B-V1.0,
🟢,CodeLlama-34b,34.0,37.42,15.1,16384,UNK,45.11,40.19,41.66,41.42,40.43,31.4,15.27,33.89,37.49,22.71,16.94,38.73,35.28,0.0,69957.0,CodeLlama-34b,https://huggingface.co/codellama/CodeLlama-34b-hf,
🟢,CodeLlama-34b-Python,34.0,36.65,15.1,16384,UNK,53.29,39.46,44.72,39.09,39.78,31.37,17.29,33.87,31.9,22.35,13.19,39.67,34.3,0.0,69957.0,CodeLlama-34b-Python,https://huggingface.co/codellama/CodeLlama-34b-Python-hf,
🔶,WizardCoder-15B-V1.0,15.0,35.0,43.7,8192,86,58.12,35.77,41.91,38.95,39.34,33.98,12.14,32.07,27.85,22.53,13.39,33.74,27.06,1470.0,32414.0,WizardCoder-15B-V1.0,https://huggingface.co/WizardLM/WizardCoder-15B-V1.0,
🔶,CodeLlama-13b-Instruct,13.0,34.04,25.3,16384,UNK,50.6,33.99,40.92,36.36,32.07,32.23,16.29,31.29,31.6,20.14,16.66,32.82,31.75,0.0,28568.0,CodeLlama-13b-Instruct,https://huggingface.co/codellama/CodeLlama-13b-Instruct-hf,
🟢,CodeGemma-7B,7.0,33.83,,8192,,40.13,35.03,43.06,40.34,34.76,28.91,10.44,30.6,32.88,20.96,13.55,34.56,32.62,,,CodeGemma-7B,https://huggingface.co/google/codegemma-7b,
🟢,CodeLlama-13b,13.0,31.96,25.3,16384,UNK,35.07,32.23,38.26,35.81,32.57,28.01,15.78,28.35,31.26,18.32,13.63,29.72,29.54,0.0,28568.0,CodeLlama-13b,https://huggingface.co/codellama/CodeLlama-13b-hf,
🟢,CodeLlama-13b-Python,13.0,29.88,25.3,16384,UNK,42.89,33.56,40.66,36.21,34.55,30.4,9.82,28.67,29.9,18.35,12.51,29.32,25.85,0.0,28568.0,CodeLlama-13b-Python,https://huggingface.co/codellama/CodeLlama-13b-Python-hf,
🟢,StarCoder2-7B,7.0,28.25,,16384,17,34.09,29.42,35.35,33.63,30.58,20.42,15.12,26.1,30.67,16.72,11.58,29.62,26.06,,,StarCoder2-7B,https://huggingface.co/bigcode/starcoder2-7b,
🔶,CodeLlama-7b-Instruct,7.0,28.15,33.1,16384,UNK,45.65,28.77,33.11,29.03,28.55,27.58,11.81,26.45,30.47,19.7,11.81,24.27,26.66,693.0,15853.0,CodeLlama-7b-Instruct,https://huggingface.co/codellama/CodeLlama-7b-Instruct-hf,
🔶,CodeGemma-7B-it,7.0,27.0,,8192,,52.74,41.42,47.71,42.6,19.18,19.56,11.83,27.76,22.02,21.64,9.22,38.31,6.91,,,CodeGemma-7B-it,https://huggingface.co/google/codegemma-7b-it,
🔶 EXT,CodeShell-7B,7.0,26.62,33.9,8194,24,34.32,30.43,33.17,28.21,30.87,22.08,8.85,24.74,22.39,20.52,17.2,24.55,24.3,639.0,18511.0,CodeShell-7B,https://huggingface.co/WisdomShell/CodeShell-7B,https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard/discussions/16
🟢,CodeLlama-7b,7.0,26.54,33.1,16384,UNK,29.98,29.2,31.8,27.23,25.17,25.6,11.6,24.36,30.36,18.04,11.94,25.82,25.52,693.0,15853.0,CodeLlama-7b,https://huggingface.co/codellama/CodeLlama-7b-hf,
🔶,OctoCoder-15B,15.0,25.0,44.4,8192,86,45.3,26.03,32.8,29.32,26.76,24.5,13.35,24.01,22.56,14.39,10.61,24.26,18.24,1520.0,32278.0,OctoCoder-15B,https://huggingface.co/bigcode/octocoder,
🟢,CodeLlama-7b-Python,7.0,24.77,33.1,16384,UNK,40.48,29.15,36.34,30.34,1.08,28.53,8.94,23.5,26.15,18.25,9.04,26.96,26.75,693.0,15853.0,CodeLlama-7b-Python,https://huggingface.co/codellama/CodeLlama-7b-Python-hf,
🟢,StarCoder-15B,15.0,24.35,43.9,8192,86,33.57,30.22,30.79,31.55,26.08,23.02,13.57,22.74,23.89,15.5,0.07,21.84,22.74,1490.0,33461.0,StarCoder-15B,https://huggingface.co/bigcode/starcoder,
🟢,Falcon-180B,180.0,24.2,,2048,,35.37,28.48,31.68,28.57,,24.53,14.1,24.08,26.71,,10.56,25.0,15.82,,,Falcon-180B,https://huggingface.co/tiiuae/falcon-180B,
🟢,StarCoderBase-15B,15.0,23.69,43.8,8192,86,30.35,28.53,31.7,30.56,26.75,21.09,10.01,22.4,26.61,10.18,11.77,24.46,16.74,1460.0,32366.0,StarCoderBase-15B,https://huggingface.co/bigcode/starcoderbase,
🟢,StarCoder2-3B,3.0,22.75,,16384,17,31.44,27.41,35.37,27.24,27.61,19.87,12.56,23.43,28.01,14.22,7.8,24.52,25.09,,,StarCoder2-3B,https://huggingface.co/bigcode/starcoder2-3b,
🟢,CodeGeex2-6B,6.0,20.35,32.7,8192,100,33.49,23.46,29.9,28.45,25.27,20.93,8.44,21.23,15.94,14.58,11.75,20.45,22.06,982.0,14110.0,CodeGeex2-6B,https://huggingface.co/THUDM/codegeex2-6b,
🟢,StarCoderBase-7B,7.0,19.85,46.9,8192,86,28.37,24.44,27.35,23.3,22.12,21.77,8.1,20.17,23.35,14.51,11.08,22.6,15.1,1700.0,16512.0,StarCoderBase-7B,https://huggingface.co/bigcode/starcoderbase-7b,
🔶,OctoGeeX-7B,7.0,19.58,32.7,8192,100,42.28,19.33,28.5,23.93,25.85,22.94,9.77,20.79,16.19,13.66,12.02,17.94,17.03,982.0,14110.0,OctoGeeX-7B,https://huggingface.co/bigcode/octogeex,
🔶,WizardCoder-3B-V1.0,3.0,18.35,50.0,8192,86,32.92,24.34,26.16,24.94,24.83,19.6,7.91,20.15,21.75,13.64,9.44,20.56,15.7,1770.0,8414.0,WizardCoder-3B-V1.0,https://huggingface.co/WizardLM/WizardCoder-3B-V1.0,
🟢,CodeGen25-7B-multi,7.0,17.88,32.6,2048,86,28.7,26.01,26.27,25.75,21.98,19.11,8.84,20.04,23.44,11.59,10.37,21.84,16.62,680.0,15336.0,CodeGen25-7B-multi,https://huggingface.co/Salesforce/codegen25-7b-multi,
🟢,CodeGemma-2B,2.0,17.5,,8192,,27.28,24.71,29.94,29.33,28.76,0.01,9.54,18.95,25.23,10.71,8.77,26.81,6.3,,,CodeGemma-2B,https://huggingface.co/google/codegemma-2b,
🔶,Refact-1.6B,1.6,16.92,50.0,4096,19,31.1,22.78,22.36,21.12,22.36,13.84,10.26,17.86,15.53,13.04,4.97,18.59,18.35,2340.0,5376.0,Refact-1.6B,https://huggingface.co/smallcloudai/Refact-1_6B-fim,
🟢,Stable-code-3b,3.0,16.17,,16384,18,30.72,28.75,31.64,29.42,23.68,21.41,10.09,19.06,17.54,13.37,0.0,22.15,0.0,,,Stable-code-3b,https://huggingface.co/stabilityai/stable-code-3b,https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard/discussions/57
🟢 EXT,DeepSeek-Coder-1b-base,1.0,16.0,,16384,UNK,32.13,27.16,28.46,27.96,22.75,15.17,9.91,19.46,19.44,11.4,9.58,18.13,11.39,,,DeepSeek-Coder-1b-base,https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-base,https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard/discussions/33
🟢,StarCoderBase-3B,3.0,13.58,50.0,8192,86,21.5,19.25,21.32,19.43,18.55,16.1,4.97,15.29,18.04,10.1,7.87,16.32,9.98,1770.0,8414.0,StarCoderBase-3B,https://huggingface.co/bigcode/starcoderbase-3b,
🔶,WizardCoder-1B-V1.0,1.1,12.12,71.4,8192,86,23.17,19.68,19.13,15.94,14.71,13.85,4.64,13.89,15.52,10.01,6.51,13.91,9.59,2360.0,4586.0,WizardCoder-1B-V1.0,https://huggingface.co/WizardLM/WizardCoder-1B-V1.0,
🟢,Replit-2.7B,2.7,10.15,42.2,2048,20,20.12,21.39,20.18,20.37,16.14,1.24,6.41,11.62,2.11,7.2,3.22,15.19,5.88,577.0,7176.0,Replit-2.7B,https://huggingface.co/replit/replit-code-v1-3b,
🟢,CodeGen25-7B-mono,7.0,9.77,34.1,2048,86,33.08,19.75,23.22,18.62,16.75,4.65,4.32,12.1,6.75,4.41,4.07,7.83,1.71,687.0,15336.0,CodeGen25-7B-mono,https://huggingface.co/Salesforce/codegen25-7b-mono,
🟢,StarCoderBase-1.1B,1.1,9.73,71.4,8192,86,15.17,14.2,13.38,11.68,9.94,11.31,4.65,9.81,12.52,5.73,5.03,10.24,3.92,2360.0,4586.0,StarCoderBase-1.1B,https://huggingface.co/bigcode/starcoderbase-1b,
🟢,CodeGen-16B-Multi,16.0,8.62,17.2,2048,6,19.26,22.2,19.15,21.0,8.37,0.0,7.68,9.89,8.5,6.45,0.66,4.21,1.25,0.0,32890.0,CodeGen-16B-Multi,https://huggingface.co/Salesforce/codegen-16B-multi,
🟢,StableCode-3B-alpha,3.0,7.58,30.2,16384,7,20.2,19.54,18.98,20.77,3.95,0.0,4.77,8.1,5.14,0.8,0.008,2.03,0.98,718.0,15730.0,StableCode-3B-alpha,https://huggingface.co/stabilityai/stablecode-completion-alpha-3b,
🟢,DeciCoder-1B,1.0,7.35,54.6,2048,3,19.32,15.3,17.85,6.87,2.01,0.0,6.08,5.86,0.0,0.1,0.47,1.72,0.63,2490.0,4436.0,DeciCoder-1B,https://huggingface.co/Deci/DeciCoder-1b,
🟢,Phi-1,1.0,6.92,,2048,1,51.22,10.76,19.25,14.29,12.42,0.63,7.05,12.15,6.21,6.21,3.11,4.49,10.13,,4941.0,Phi-1,https://huggingface.co/microsoft/phi-1,
🟢,SantaCoder-1.1B,1.1,5.96,50.8,2048,3,18.12,15.0,15.47,6.2,1.5,0.0,0.0,4.92,0.1,0.0,0.0,2.0,0.7,2270.0,4602.0,SantaCoder-1.1B,https://huggingface.co/bigcode/santacoder,
|