File size: 8,740 Bytes
62c8319
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
Model,NUM_Q_multich_EM,NUM_Q_multich_CC,NUM_Q_multich_PM,NUM_Q_onech_EM,NUM_Q_onech_CC,NUM_Q_onech_PM,NUM_Q_seq_EM,NUM_Q_seq_CC,NUM_Q_seq_PM,NUM_Q_map_EM,NUM_Q_map_CC,NUM_Q_map_PM,OPEN_Q_EM,OPEN_Q_F1,OPEN_Q_LR,Q_TYPE_avg
claude-3.5-sonnet,84.2047930283224,87.037037037037,91.557734204793,96.0132890365448,96.0132890365448,96.0132890365448,70.6185567010309,70.6185567010309,71.1340206185567,46.7811158798283,46.7811158798283,48.9270386266094,64.9484536082474,72.1616173155497,81.745704467354,74.9703740785215
gpt-4o,73.6383442265795,78.322440087146,84.2592592592593,94.4352159468439,94.5182724252492,94.5182724252492,60.8247422680412,61.340206185567,62.8865979381443,18.0257510729614,18.4549356223176,24.2489270386266,13.4020618556701,65.2801523778733,75.1821305841924,61.28915395424806
gemma2:27b-instruct-q4_0,1.44711640774633,6.89508406043839,36.7950627793147,22.1528861154446,63.0265210608424,46.628531807939,2.53968253968254,8.57142857142857,8.57142857142857,0.0,0.0828500414250207,0.787075393537697,1.64661177960735,8.23779148465489,15.1519949335022,14.835604369799482
gemma2:9b-instruct-q4_0,40.6469461587572,52.6282187699511,63.8540114918068,76.5817299358641,79.0084936730803,76.5990639625585,29.3121693121693,30.4761904761905,35.026455026455,4.30820215410108,4.88815244407622,9.48632974316487,24.4458518049398,32.0378038500143,49.7612412919569,40.604057339672394
gemma:7b-instruct-v1.1-q4_0,2.40476697169611,12.1302404766972,30.9640349010428,27.2837580169873,53.5794765123938,53.4408042988386,4.97354497354497,11.7460317460317,5.60846560846561,0.828500414250207,0.828500414250207,2.52692626346313,0.0633312222925902,6.03356941864105,23.6782773907536,15.739348575289926
ilyagusev/saiga_llama3,2.08555011704618,10.0021281123643,47.3824217918706,63.0438550875368,64.3092390362281,64.2052348760617,6.87830687830688,6.87830687830688,18.0952380952381,0.0,0.0,2.8169014084507,6.90310322989234,21.9392109592531,40.68397720076,23.681564911421045
llama-405b,10.6753812636166,35.29411765,65.95860566,0.913621262458472,85.9634551495017,73.7541528239203,1.54639175257732,9.27835051546392,27.5773195876289,0.858369098712446,4.29184549356223,39.6995708154506,4.46735395189003,43.3361397687549,58.5773195876289,30.81279962541108
llama2:13b,0.0,0.0425622472866567,28.2400510746967,0.0,41.0469752123418,1.73340266944011,0.0,0.105820105820106,2.75132275132275,0.0,0.0,0.124275062137531,0.0633312222925902,5.16796089780207,13.0633312222926,6.155935497695527
llama3.1:70b-instruct-q4_0,26.4311555650138,50.3298574164716,61.5450095765056,24.284971398856,82.6833073322933,81.1232449297972,8.35978835978836,15.6613756613757,26.5608465608466,1.49130074565037,2.23695111847556,10.149130074565,14.3128562381254,48.89922596918,63.82900569981,34.52653510978363
llama3.1:8b-instruct-q4_0,0.0638433709299851,0.617152585656523,46.7439880825708,8.40700294678454,59.4037094817126,48.6566129311839,0.0,1.90476190476191,6.34920634920635,0.0,0.0,4.63960231980116,0.443318556048132,19.2284319751355,39.6263457884737,15.738931752817674
llama3:70b-instruct-q4_0,33.4964886145989,59.8638008086827,59.1828048520962,78.1417923383602,79.9791991679667,78.1591263650546,16.6137566137566,17.8835978835979,28.4126984126984,4.14250207125104,5.2195526097763,7.24937862468931,10.3229892336922,44.8346410536552,60.4901836605447,38.93283415402807
llama3:8b-instruct-q4_0,0.319216854649925,1.00021281123643,47.4037029155139,26.4863927890449,64.1012307158953,63.5812099150633,0.529100529100529,1.48148148148148,4.07407407407407,0.0,0.0828500414250207,2.27837613918807,0.0633312222925902,17.1091465760483,35.2742241925269,17.58563668383605
mistral-123b,42.26579521,64.27015251,61.87363834,0.0,89.0365448504983,53.2392026578073,0.0,82.98969072,0.0,0.858369099,27.46781116,3.433476395,21.64948454,39.29104393,49.30584192,35.712070088820376
mistral:7b-instruct-v0.3-q4_0,0.0,0.0212811236433284,44.5520323473079,46.6458658346334,58.0689894262437,56.612931183914,0.0,0.0,26.984126984127,0.0,0.0,3.93537696768848,2.46991766941102,11.6013570437946,26.0025332488917,18.45962745531034
mixtral:8x7b-instruct-v0.1-q4_0,0.0,11.8110236220472,51.0534156203448,8.66701334720055,77.8124458311666,67.2733576009707,0.105820105820106,16.9312169312169,19.4708994708995,0.0,1.65700082850041,10.4805302402651,0.633312222925902,5.19230683158126,9.84547181760608,18.72892096470301
phi3:14b-medium-4k-instruct-q4_0,0.0425622472866567,0.148967865503299,57.1398169823367,56.6302652106084,77.2404229502513,72.2828913156526,2.75132275132275,4.86772486772487,36.2962962962963,0.0,0.331400165700083,9.07207953603977,5.25649145028499,16.8528380209838,31.6485117162761,24.704106091751175
qwen2:72b-instruct-q4_0,55.5862949563737,69.9723345392637,71.8769951053416,85.6647599237303,89.3222395562489,85.9594383775351,62.010582010582,62.1164021164021,62.1164021164021,31.4001657000828,34.3827671913836,36.8682684341342,8.92970234325522,43.0269331550944,57.1019632678911,57.08901658624805
qwen2:7b-instruct-q4_0,10.3851883379442,13.024047669717,54.7137688869972,66.2159819726122,67.8973825619691,67.0653492806379,19.5767195767196,20.5291005291005,21.2698412698413,1.90555095277548,2.31980115990058,7.49792874896438,3.73654211526282,11.4397586578317,32.7891070297657,26.691071250002647
qwen:7b,0.0,0.0212811236433284,30.0915088316663,37.0948171260184,37.857514300572,37.0948171260184,8.99470899470899,10.0529100529101,11.957671957672,0.0,0.0,1.32560066280033,0.0,2.44002256870951,17.0981633945535,12.935267742618189
solar:10.7b-instruct-v1-q4_0,0.0,0.25537348371994,48.669929772292,46.2991853007454,64.2745709828393,62.2811579129832,0.0,0.211640211640212,18.2539682539683,0.0,0.0828500414250207,3.23115161557581,3.41988600379987,11.4318388715363,23.1849271690944,18.773098641307982
wavecut/vikhr:7b-instruct_0.4-Q4_1,0.0,0.0212811236433284,36.1247073845499,34.3213728549142,42.9883862021148,40.4402842780378,0.952380952380952,2.75132275132275,6.24338624338624,0.0,0.0,0.579950289975145,10.0063331222293,19.0998530939532,34.2970234325522,15.188418781937322
yi:6b,0.595871462013194,8.93807193019791,21.5896999361566,12.619171433524,44.6697867914717,26.7464031894609,0.317460317460317,9.73544973544974,1.74603174603175,0.165700082850041,0.828500414250207,0.414250207125104,0.189993666877771,2.29393179599174,10.265357821406,9.40771203535113
yi:9b,4.08597573951905,11.5130878910406,34.8691210895935,25.8970358814353,47.1658866354654,42.6417056682267,8.99470899470899,10.2645502645503,12.6984126984127,0.0,0.248550124275062,1.40845070422535,0.633312222925902,3.4585674314986,15.136795440152,14.601077385735296
GigaChat_Lite,51.19825708061,55.5555555555556,68.0283224400871,69.2691029900332,71.0963455149502,69.2691029900332,34.020618556701,34.020618556701,34.020618556701,6.00858369098712,6.00858369098712,8.15450643776824,32.6460481099656,40.3723509295672,58.9209621993127,42.57263848666402
GigaChat_Plus,51.19825708061,55.5555555555556,68.0283224400871,69.2691029900332,71.0963455149502,69.2691029900332,34.020618556701,34.020618556701,34.020618556701,6.00858369098712,6.00858369098712,8.15450643776824,32.9896907216495,40.1954759333432,58.8831615120275,42.58123628187567
GigaChat_Pro,63.2897603485839,66.2309368191721,75.4357298474945,76.3289036544851,77.7408637873754,76.3289036544851,52.0618556701031,52.0618556701031,52.0618556701031,11.587982832618,11.587982832618,15.8798283261803,40.893470790378,47.6443830947436,64.5051546391753,52.24263117584123
yandexgpt_lite,7.29847494553377,8.06100217864924,54.0849673202614,76.1627906976744,77.3255813953489,76.6611295681063,19.5876288659794,19.5876288659794,24.4845360824742,1.28755364806867,1.28755364806867,6.65236051502146,31.9587628865979,49.8317287269818,64.9347079037801,34.61376048323505
yandexgpt_pro,47.2766884531591,49.3464052287582,75.0544662309368,84.8837209302326,85.0498338870432,84.8837209302326,41.7525773195876,41.7525773195876,43.298969072165,5.5793991416309,5.5793991416309,7.29613733905579,15.4639175257732,50.7635115107509,63.6975945017182,46.77859456881752
AnatoliiPotapov/T-lite-instruct-0.1,0.0,7.29847494553376,52.1786492374727,0.0,83.7209302325581,74.8338870431893,0.0,17.0103092783505,17.0103092783505,0.0,0.858369098712446,2.57510729613733,0.0,4.52000213091168,9.50859106529209,17.967641973767225
random,4.04341349223239,7.59736114066823,32.698446477974,24.5103137458832,24.5103137458832,24.5103137458832,14.0740740740741,14.0740740740741,14.0740740740741,0.828500414250207,0.828500414250207,3.23115161557581,,,,13.748378084568563
Среднее значение,16.24417873849718,23.5159131873249,50.93332969343993,40.9919790637536,66.42771398247166,59.83125300332374,13.213350632349158,19.17732270450974,21.394109763034663,2.759259419186267,4.1824412773457125,7.498150986205318,10.133664587126296,23.93628984001529,38.04673218695349,26.478875574939632