| Model,GEO_num_q_EM,GEO_num_q_CC,GEO_num_q_PM,GEO_open_q_EM,GEO_open_q_F1,GEO_open_q_LR,GEO_avg,HIST_num_q_EM,HIST_num_q_CC,HIST_num_q_PM,HIST_open_q_EM,HIST_open_q_F1,HIST_open_q_LR,HIST_avg,SOC_num_q_EM,SOC_num_q_CC,SOC_num_q_PM,SOC_open_q_EM,SOC_open_q_F1,SOC_open_q_LR,SOC_avg,POL_num_q_EM,POL_num_q_CC,POL_num_q_PM,POL_avg,DOMAIN_avg | |
| claude-3.5-sonnet,91.324200913242,91.7808219178082,92.351598173516,69.6969696969697,77.7833237816263,83.6666666666667,84.43393019163814,81.2297734627832,81.5533980582524,82.9557713052859,57.2192513368984,65.2335005485077,77.9197860962567,74.35191346799739,87.3804971319312,89.3881453154876,92.0172084130019,94.7368421052632,96.4912280701754,97.2368421,92.87512718930988,78.2608695652174,78.2608695652174,81.5217391304348,79.34782608695654,83.2385382549782 | |
| gpt-4o,80.5936073059361,81.0502283105023,82.5342465753425,3.03030303030303,71.3585280061966,75.7878787878788,65.72579866935989,71.0895361380798,72.168284789644,75.2966558791802,16.5775401069519,58.3967637801363,72.1283422459893,60.942853823330246,82.1223709369025,85.3728489483748,88.4799235181644,15.7894736842105,88.5964912280702,89.1578947,74.91983383595372,73.9130434782609,73.9130434782609,76.0869565217391,74.6376811594203,68.25923625952971 | |
| gemma2:27b-instruct-q4_0,15.5348837209302,36.2325581395349,34.7209302325581,1.36518771331058,6.08108161580778,12.3447098976109,17.71322521995874,9.0968443960827,29.5756256800871,26.822633297062,1.97309417040359,8.93722159294191,16.0089686098655,15.402397957740467,8.88761467889908,29.1284403669725,42.6032110091743,0.0,7.37259511525245,14.374269005848,17.061021696024387,23.7947122861586,59.0979782270607,46.8895800933126,43.26075686884396,20.51629237375588 | |
| gemma2:9b-instruct-q4_0,58.9302325581395,62.3720930232558,64.4883720930233,30.7167235494881,38.6347348301442,54.7849829351536,51.65452316486741,39.8694232861806,43.4385201305767,48.139281828074,17.6681614349776,25.4223580804077,44.1973094170404,36.45584236287616,59.6139143730887,68.4633027522936,72.887996941896,57.8947368421053,63.8700918964077,77.4327485380117,66.6937985573005,68.2737169517885,71.3841368584759,70.4510108864697,70.0362882322447,54.234945200333264 | |
| gemma:7b-instruct-v1.1-q4_0,20.8372093023256,39.2093023255814,42.046511627907,0.0,3.94542088889186,19.0238907849829,20.843722488281458,9.88030467899891,19.8258977149075,23.8737758433079,0.0896860986547085,4.38105088097796,23.3417040358744,13.565403208786895,12.6911314984709,32.0336391437309,43.243501529052,0.0,20.3867021011607,33.8479532163743,23.70048791479813,27.6827371695179,54.7433903576983,55.6765163297045,46.03421461897357,23.179063120386648 | |
| ilyagusev/saiga_llama3,37.8139534883721,40.8372093023256,49.3023255813954,15.358361774744,35.1715192074092,52.7337883959044,38.53619295835845,26.2241566920566,27.7040261153428,37.1490750816104,2.51121076233184,15.1855722725022,34.7470852017937,23.920187687606255,26.5863914373089,32.2438837920489,56.5271406727829,21.0526315789474,43.3031923570162,58.7485380116959,39.74362964163337,60.8087091757387,63.2970451010886,63.9191290824261,62.67496111975114,38.15356881356391 | |
| llama-405b,35.8447489,70.0913242009132,77.3972602739726,1.51515151515152,61.7983715080829,68.0454545454545,52.44871849059579,38.9428263,58.7918015102481,68.0690399137001,10.1604278074866,49.7459710572709,67.7005347593583,48.901766891344,39.0057361,76.3862332695985,84.6558317399618,7.89473684210526,78.2456140350877,81.6578947368421,61.307674453932556,46.3768115942029,74.6376811594203,76.8115942028985,65.94202898550724,55.89404980817884 | |
| llama2:13b,0.0,31.6279069767442,4.69767441860465,0.341296928327645,9.40460676527417,20.5904436860068,11.11032146249291,0.0,19.0424374319913,5.61479869423286,0.0,4.03632452572392,11.5237668161435,6.702887911348597,0.0,9.53746177370031,20.3172782874618,0.0,5.28747736387109,10.2046783625731,7.55781596460105,0.0,49.3001555209953,5.05443234836703,18.11819595645411,9.837178090477034 | |
| llama3.1:70b-instruct-q4_0,17.2558139534884,60.6511627906977,65.3255813953489,5.11945392491468,59.2786046255473,67.2184300341297,45.80817445402112,16.7791077257889,46.8335146898803,51.2622415669206,16.1434977578475,43.4389491770808,61.27533632287,39.288774540064686,28.3448012232416,64.7362385321101,71.1678134556575,18.1286549707602,66.7182357754662,74.672514619883,53.96137642951976,17.8849144634526,72.6283048211509,71.850699844479,54.12130637636083,47.46256531765313 | |
| llama3.1:8b-instruct-q4_0,4.6046511627907,31.8139534883721,32.1860465116279,0.0,32.1790002422677,49.0546075085324,24.973043152265134,3.13384113166485,24.3960826985854,30.4678998911861,0.62780269058296,14.4408672828017,36.1085201793722,18.195835645698867,3.26834862385321,24.8853211009174,51.5768348623853,0.0,28.2554385814658,46.4093567251462,25.732549982294653,11.5085536547434,57.0762052877138,51.1664074650078,39.917055469155,25.388559004238896 | |
| llama3:70b-instruct-q4_0,57.3953488372093,63.4883720930233,63.9069767441861,5.80204778156997,60.7482833489855,68.3788395904437,53.28664473256964,40.7181719260065,46.5070729053319,49.2491838955386,10.6726457399103,40.3680070949795,58.322869955157,40.9729919194873,52.3891437308869,70.7759938837921,67.8516819571865,15.7894736842105,46.6919490734894,61.1052631578947,52.43391758124334,68.895800933126,70.6065318818041,71.3841368584759,70.29548989113533,51.95465690824798 | |
| llama3:8b-instruct-q4_0,15.8139534883721,35.5348837209302,45.7674418604651,0.0,31.6684961159729,48.0511945392492,29.472661620831584,10.5114254624592,25.5495103373232,33.8737758433079,0.0896860986547085,12.4341412479717,31.7417040358744,19.033373837598518,11.2767584097859,27.1215596330275,57.7025993883792,0.0,22.6456467258,36.4152046783626,25.860294805892533,20.99533437014,62.6749611197512,62.5194401244168,48.729911871436,28.20893891429732 | |
| mistral-123b,78.310502283105,78.5388127853881,80.5936073059361,59.0909090909091,70.4508879111781,77.9090909090909,74.14896838093455,66.235167206041,66.6666666666667,69.848975188781,42.2459893048128,50.9433883012481,69.5989304812834,60.92318619147216,76.6730401529637,80.1147227533461,84.5602294455067,89.4736842105263,91.2280701754386,93.7368421052632,85.96443147384078,69.5652173913043,69.5652173913043,71.7391304347826,70.28985507246374,73.19471816642266 | |
| mistral:7b-instruct-v0.3-q4_0,25.4883720930233,29.953488372093,44.4883720930233,6.8259385665529,14.7143576600266,29.5290102389079,25.166589837271164,18.759521218716,23.8302502720348,34.7007616974973,0.717488789237668,8.34350531701643,24.0941704035874,18.407616283014935,18.4059633027523,23.7576452599388,52.3318042813456,6.4327485380117,27.5101026274299,32.4035087719298,26.806962130234684,49.4556765163297,57.3872472783826,58.0093312597201,54.950751684810804,27.959012597978905 | |
| mixtral:8x7b-instruct-v0.1-q4_0,5.53488372093023,48.7906976744186,52.3953488372093,0.341296928327645,3.94842331452323,6.73037542662116,19.62350431700503,3.5038084874864,37.9542981501632,42.3612622415669,0.717488789237668,4.68070314515325,10.1865470852018,16.5673513164682,3.45948012232416,37.5955657492355,59.7094801223242,0.584795321637427,10.6595347898574,12.9590643274854,20.827986738810683,6.22083981337481,72.1617418351477,61.1975116640747,46.52669777086573,22.937768930776222 | |
| phi3:14b-medium-4k-instruct-q4_0,34.2325581395349,43.8139534883721,59.5813953488372,12.9692832764505,30.2658027774082,44.8976109215017,37.62676732535076,24.069640914037,33.6235038084875,46.9314472252448,2.69058295964126,12.3929007725114,27.7345291479821,24.573767471317343,21.5596330275229,30.2752293577982,65.7874617737003,8.7719298245614,22.9512670175593,34.4678362573099,30.63555954307533,50.5443234836703,68.7402799377916,66.5629860031104,61.94919647485742,35.37448359347776 | |
| qwen2:72b-instruct-q4_0,80.5581395348837,83.953488372093,83.7441860465116,1.70648464163823,60.9664006973775,66.8259385665529,62.95910630984282,60.1088139281828,65.310119695321,65.4733405875952,10.6726457399103,37.226795170081,53.6,48.73195252018172,68.5015290519878,79.434250764526,78.3925840978593,9.94152046783626,50.1081604264453,63.2748538011696,58.27548310163737,68.5847589424572,74.805598755832,71.2286158631415,71.53965785381023,58.781820245304864 | |
| qwen2:7b-instruct-q4_0,45.7209302325581,46.9302325581395,54.8837209302326,6.48464163822526,12.4030115766671,33.8430034129693,33.37759005813198,27.8346028291621,30.1196953210011,37.2143634385201,2.0627802690583,7.87949319222932,29.6869955156951,22.466321760944336,35.4548929663609,37.2133027522936,65.5485474006116,9.94152046783626,33.0038691193979,51.2105263157895,38.72877650371496,62.0528771384137,63.9191290824261,64.0746500777605,63.34888543286676,36.0706088683499 | |
| qwen:7b,20.093023255814,20.8837209302326,26.5813953488372,0.0,1.97488246634156,16.8600682593857,14.39884837676851,14.6681175190424,15.4080522306855,21.7627856365615,0.0,1.34680160823358,15.7542600896861,11.49000284736818,16.532874617737,16.5519877675841,35.3211009174312,0.0,10.3653291238234,26.2690058479532,17.50671637908815,39.50233281493,39.9688958009331,40.9797822706065,40.15033696215653,18.134496024086605 | |
| solar:10.7b-instruct-v1-q4_0,23.7674418604651,34.4651162790698,46.0232558139535,1.70648464163823,8.4900835179791,17.1194539249147,21.92863933967007,16.474428726877,26.89880304679,37.0729053318825,3.04932735426009,10.4661683783297,22.9569506726457,19.48643058513083,21.6360856269113,25.9556574923547,57.559250764526,8.7719298245614,22.7690138336274,35.0643274853801,28.626044171226813,42.1461897356143,60.3421461897356,60.1866251944012,54.22498703991704,27.7581736045675 | |
| wavecut/vikhr:7b-instruct_0.4-Q4_1,17.8139534883721,23.2093023255814,28.093023255814,12.9692832764505,22.0442276930355,35.7849829351536,23.31912882906785,14.2110990206746,17.5843307943417,23.4820457018498,7.98206278026906,16.8323787548876,32.647533632287,18.78990844738496,14.2201834862385,17.6796636085627,40.9690366972477,18.1286549707602,28.8398070736436,42.5029239766082,27.05671163551015,32.5038880248834,42.7682737169518,41.9906687402799,39.08761016070503,25.34558685494728 | |
| yi:6b,7.06976744186047,26.4651162790698,17.4651162790698,0.0,1.72191372616652,8.90102389078498,10.270489602825263,3.65614798694233,22.6550598476605,13.7540805223069,0.0,2.08897265807147,10.7318385650224,8.814349930000601,6.19266055045872,23.9296636085627,26.3379204892966,1.75438596491228,4.61048579154667,9.56140350877193,12.064419985591483,18.1959564541213,36.8584758942457,30.0933125972006,28.382581648522535,12.954442955051032 | |
| yi:9b,16.046511627907,27.1162790697674,30.5813953488372,0.0,3.15355705067136,12.098976109215,14.832786534399661,9.07508161044614,19.7606093579978,22.829162132753,0.179372197309417,2.8367657031748,15.2224215246637,11.65056875439081,15.1567278287462,29.6062691131498,42.4120795107034,4.67836257309942,8.03562572777585,19.7836257309942,19.945448414078147,33.5925349922239,50.0777604976672,48.2115085536547,43.9606013478486,19.5454583933694 | |
| GigaChat_Lite,61.1872146118721,62.5570776255708,63.1278538812785,40.9090909090909,51.7079785604165,65.8030303030303,57.54870764854318,40.8845738942826,43.042071197411,45.5231930960086,26.2032085561497,33.7021080330654,53.9144385026738,40.54493221326518,63.4799235181644,66.5391969407266,73.565965583174,50.0,53.5087719298246,71.6052631578947,63.11652018829738,52.8985507246377,55.7971014492754,54.7101449275362,54.46859903381644,53.841274162003984 | |
| GigaChat_Plus,61.1872146118721,62.5570776255708,63.1278538812785,40.9090909090909,51.7298973967309,65.7424242424243,57.542259777827915,40.8845738942826,43.042071197411,45.5231930960086,26.7379679144385,33.4191279951085,53.8449197860963,40.57530898055759,63.4799235181644,66.5391969407266,73.565965583174,50.0,53.5087719298246,71.7631578947368,63.142835977771064,52.8985507246377,55.7971014492754,54.7101449275362,54.46859903381644,53.85562978658994 | |
| GigaChat_Pro,71.2328767123288,72.1461187214612,72.6027397260274,53.030303030303,62.0469396605864,73.8484848484849,67.48457711653195,49.8381877022654,51.024811218986,54.1531823085221,33.6898395721925,40.7455479303299,59.379679144385,48.138541312780156,74.3785850860421,76.7686424474187,81.357552581262,55.2631578947369,56.578947368421,73.5,69.64114756298011,55.072463768116,57.9710144927536,56.8840579710145,56.642512077294704,61.02443486598271 | |
| yandexgpt_lite,47.2602739726027,48.1735159817352,55.5936073059361,42.4242424242424,55.6932275713253,67.6363636363636,52.79687181536755,39.697950377562,40.453074433657,50.9708737864078,28.3422459893048,42.8268094822329,61.0160427807487,43.884499474985525,35.3728489483748,36.2332695984704,65.2485659655832,31.5789473684211,74.1228070175439,79.5263157894737,53.68045911464452,57.9710144927536,58.695652173913,60.8695652173913,59.17874396135263,51.41462925304969 | |
| yandexgpt_pro,63.9269406392694,64.3835616438356,68.7214611872146,22.7272727272727,44.3545700753268,56.3939393939394,53.41795761114307,56.0949298813376,56.2028047464941,62.2977346278317,7.48663101604278,47.2091634117127,61.855614973262,48.52447977611348,63.6711281070746,65.2963671128107,80.9273422562142,42.1052631578947,79.3859649122807,85.4473684210526,69.47223899455459,60.8695652173913,61.5942028985507,63.0434782608696,61.83574879227053,57.809300222270394 | |
| AnatoliiPotapov/T-lite-instruct-0.1,0.0,49.7716894977168,55.2511415525114,0.0,5.98577838322242,11.4848484848484,20.415576319716504,0.0,42.0711974110032,47.8964401294498,0.0,4.86769650696588,10.2192513368983,17.509097564052862,0.0,38.4321223709369,61.4244741873805,0.0,0.00263157894736842,2.57894736842105,17.073029250947638,0.0,72.463768115942,65.5797101449275,46.014492753623166,22.287128431865312 | |
| random,16.8372093023256,17.5813953488372,22.3720930232558,,,,18.930232558139533,13.2535364526659,13.9281828073993,20.4134929270947,,,,15.8650707290533,11.6207951070336,13.8188073394495,29.0997706422018,,,,18.17979102956163,26.1275272161742,26.905132192846,28.149300155521,27.0606531881804,20.008936876233715 | |
| Среднее значение,37.07388057198435,49.53268202895473,52.6650844217904,15.001373033619705,34.127721068109985,45.20998320262767,38.72651861212592,28.224186428336843,38.16539247552274,42.49944575717629,11.258987421950518,24.132036341436354,39.56758797647291,30.489253847078714,34.04543277210754,45.86048764966486,60.77167180248819,21.334946561806824,41.20875250919483,51.273038917685014,42.08640300826897,42.553249036446005,60.44796808438704,57.71840557170868,53.573207564180585,39.4875165299323 | |