| {"tatdqa_test": {"ndcg_at_1": 0.49818, "ndcg_at_3": 0.6076, "ndcg_at_5": 0.63964, "ndcg_at_10": 0.67349, "ndcg_at_20": 0.68939, "ndcg_at_100": 0.7009, "ndcg_at_1000": 0.70248, "map_at_1": 0.49818, "map_at_3": 0.58111, "map_at_5": 0.59875, "map_at_10": 0.61298, "map_at_20": 0.6174, "map_at_100": 0.61912, "map_at_1000": 0.6192, "recall_at_1": 0.49818, "recall_at_3": 0.68408, "recall_at_5": 0.76245, "recall_at_10": 0.86574, "recall_at_20": 0.92831, "recall_at_100": 0.98846, "recall_at_1000": 1.0, "precision_at_1": 0.49818, "precision_at_3": 0.22803, "precision_at_5": 0.15249, "precision_at_10": 0.08657, "precision_at_20": 0.04642, "precision_at_100": 0.00988, "precision_at_1000": 0.001, "mrr_at_1": 0.49878493317132444, "mrr_at_3": 0.5813082219522074, "mrr_at_5": 0.5994734710409073, "mrr_at_10": 0.6137702752222801, "mrr_at_20": 0.6182156009116668, "mrr_at_100": 0.6199147929198118, "mrr_at_1000": 0.6199842657196033, "naucs_at_1_max": 0.05209323809834884, "naucs_at_1_std": -0.14640383640627463, "naucs_at_1_diff1": 0.6924261826062196, "naucs_at_3_max": 0.07261021252891718, "naucs_at_3_std": -0.17093277454224043, "naucs_at_3_diff1": 0.5892564383877543, "naucs_at_5_max": 0.0850814342721355, "naucs_at_5_std": -0.17505101772524606, "naucs_at_5_diff1": 0.5655965042996709, "naucs_at_10_max": 0.2860253169991708, "naucs_at_10_std": 0.020195118468011905, "naucs_at_10_diff1": 0.5470729581140661, "naucs_at_20_max": 0.3992322089812412, "naucs_at_20_std": 0.17887142347112067, "naucs_at_20_diff1": 0.5323312554612188, "naucs_at_100_max": 0.7879493626883052, "naucs_at_100_std": 0.9412662796474637, "naucs_at_100_diff1": 0.41102807146513015, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "shiftproject_test": {"ndcg_at_1": 0.64, "ndcg_at_3": 0.76571, "ndcg_at_5": 0.7825, "ndcg_at_10": 0.80008, "ndcg_at_20": 0.81019, "ndcg_at_100": 0.81175, "ndcg_at_1000": 0.81175, "map_at_1": 0.64, "map_at_3": 0.73333, "map_at_5": 0.74283, "map_at_10": 0.75093, "map_at_20": 0.7537, "map_at_100": 0.75382, "map_at_1000": 0.75382, "recall_at_1": 0.64, "recall_at_3": 0.86, "recall_at_5": 0.9, "recall_at_10": 0.95, "recall_at_20": 0.99, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.64, "precision_at_3": 0.28667, "precision_at_5": 0.18, "precision_at_10": 0.095, "precision_at_20": 0.0495, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.64, "mrr_at_3": 0.7383333333333333, "mrr_at_5": 0.7453333333333333, "mrr_at_10": 0.7534285714285714, "mrr_at_20": 0.7562032638414218, "mrr_at_100": 0.7563223114604694, "mrr_at_1000": 0.7563223114604694, "naucs_at_1_max": 0.13370162217396833, "naucs_at_1_std": -0.13328649891429342, "naucs_at_1_diff1": 0.5623642866266446, "naucs_at_3_max": 0.4246224192877924, "naucs_at_3_std": -0.04343910211999381, "naucs_at_3_diff1": 0.30809200498822137, "naucs_at_5_max": 0.37973856209150514, "naucs_at_5_std": 0.0060690943043917814, "naucs_at_5_diff1": 0.2945845004668548, "naucs_at_10_max": 0.3563958916900161, "naucs_at_10_std": -0.02726423902894093, "naucs_at_10_diff1": 0.2253034547152248, "naucs_at_20_max": 0.5541549953314738, "naucs_at_20_std": 0.35807656395891135, "naucs_at_20_diff1": 0.12278244631185926, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "syntheticDocQA_artificial_intelligence_test": {"ndcg_at_1": 0.91, "ndcg_at_3": 0.95417, "ndcg_at_5": 0.95847, "ndcg_at_10": 0.95847, "ndcg_at_20": 0.96097, "ndcg_at_100": 0.96097, "ndcg_at_1000": 0.96097, "map_at_1": 0.91, "map_at_3": 0.945, "map_at_5": 0.9475, "map_at_10": 0.9475, "map_at_20": 0.94817, "map_at_100": 0.94817, "map_at_1000": 0.94817, "recall_at_1": 0.91, "recall_at_3": 0.98, "recall_at_5": 0.99, "recall_at_10": 0.99, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.91, "precision_at_3": 0.32667, "precision_at_5": 0.198, "precision_at_10": 0.099, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.91, "mrr_at_3": 0.945, "mrr_at_5": 0.9475, "mrr_at_10": 0.9475, "mrr_at_20": 0.9481666666666666, "mrr_at_100": 0.9481666666666666, "mrr_at_1000": 0.9481666666666666, "naucs_at_1_max": 0.4401908911712813, "naucs_at_1_std": -0.3586990351696247, "naucs_at_1_diff1": 0.877840024898848, "naucs_at_3_max": 0.9346405228758099, "naucs_at_3_std": -0.2987861811391324, "naucs_at_3_diff1": 0.9346405228758099, "naucs_at_5_max": 1.0, "naucs_at_5_std": 0.5541549953314738, "naucs_at_5_diff1": 0.8692810457516413, "naucs_at_10_max": 1.0, "naucs_at_10_std": 0.5541549953314738, "naucs_at_10_diff1": 0.8692810457516413, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "syntheticDocQA_government_reports_test": {"ndcg_at_1": 0.89, "ndcg_at_3": 0.93917, "ndcg_at_5": 0.94734, "ndcg_at_10": 0.95067, "ndcg_at_20": 0.95067, "ndcg_at_100": 0.95067, "ndcg_at_1000": 0.95067, "map_at_1": 0.89, "map_at_3": 0.92833, "map_at_5": 0.93283, "map_at_10": 0.93426, "map_at_20": 0.93426, "map_at_100": 0.93426, "map_at_1000": 0.93426, "recall_at_1": 0.89, "recall_at_3": 0.97, "recall_at_5": 0.99, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.89, "precision_at_3": 0.32333, "precision_at_5": 0.198, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.89, "mrr_at_3": 0.9283333333333332, "mrr_at_5": 0.9333333333333332, "mrr_at_10": 0.9347619047619048, "mrr_at_20": 0.9347619047619048, "mrr_at_100": 0.9347619047619048, "mrr_at_1000": 0.9347619047619048, "naucs_at_1_max": 0.4759041319474276, "naucs_at_1_std": 0.03831285971995622, "naucs_at_1_diff1": 0.6963748818829997, "naucs_at_3_max": 0.6095549330843502, "naucs_at_3_std": -0.6305633364456867, "naucs_at_3_diff1": 0.693432928727046, "naucs_at_5_max": -0.1713352007469681, "naucs_at_5_std": -1.7399626517273863, "naucs_at_5_diff1": 0.35807656395891135, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "arxivqa_test_subsampled": {"ndcg_at_1": 0.728, "ndcg_at_3": 0.79347, "ndcg_at_5": 0.80896, "ndcg_at_10": 0.81873, "ndcg_at_20": 0.82674, "ndcg_at_100": 0.83724, "ndcg_at_1000": 0.8381, "map_at_1": 0.728, "map_at_3": 0.778, "map_at_5": 0.7865, "map_at_10": 0.79056, "map_at_20": 0.79271, "map_at_100": 0.79419, "map_at_1000": 0.79423, "recall_at_1": 0.728, "recall_at_3": 0.838, "recall_at_5": 0.876, "recall_at_10": 0.906, "recall_at_20": 0.938, "recall_at_100": 0.994, "recall_at_1000": 1.0, "precision_at_1": 0.728, "precision_at_3": 0.27933, "precision_at_5": 0.1752, "precision_at_10": 0.0906, "precision_at_20": 0.0469, "precision_at_100": 0.00994, "precision_at_1000": 0.001, "mrr_at_1": 0.728, "mrr_at_3": 0.7773333333333333, "mrr_at_5": 0.7849333333333333, "mrr_at_10": 0.7899904761904762, "mrr_at_20": 0.792047291305186, "mrr_at_100": 0.7934268855948629, "mrr_at_1000": 0.793474163928947, "naucs_at_1_max": 0.6908070592281116, "naucs_at_1_std": -0.12436887436887438, "naucs_at_1_diff1": 0.907886921044816, "naucs_at_3_max": 0.6583526470980818, "naucs_at_3_std": -0.07138003421873189, "naucs_at_3_diff1": 0.8392083581075349, "naucs_at_5_max": 0.6464957819597658, "naucs_at_5_std": -0.08967584438058089, "naucs_at_5_diff1": 0.84094280152035, "naucs_at_10_max": 0.618133778334029, "naucs_at_10_std": -0.1158789757037573, "naucs_at_10_diff1": 0.820221705703559, "naucs_at_20_max": 0.6280834914610988, "naucs_at_20_std": -0.01796632631547394, "naucs_at_20_diff1": 0.8358483178217527, "naucs_at_100_max": 0.6204481792716809, "naucs_at_100_std": 0.044817927170816896, "naucs_at_100_diff1": 0.8148148148147883, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "docvqa_test_subsampled": {"ndcg_at_1": 0.45455, "ndcg_at_3": 0.52853, "ndcg_at_5": 0.55582, "ndcg_at_10": 0.5775, "ndcg_at_20": 0.59108, "ndcg_at_100": 0.61824, "ndcg_at_1000": 0.62991, "map_at_1": 0.45455, "map_at_3": 0.51109, "map_at_5": 0.52616, "map_at_10": 0.53522, "map_at_20": 0.53902, "map_at_100": 0.54294, "map_at_1000": 0.54337, "recall_at_1": 0.45455, "recall_at_3": 0.57871, "recall_at_5": 0.64523, "recall_at_10": 0.71175, "recall_at_20": 0.76497, "recall_at_100": 0.90909, "recall_at_1000": 1.0, "precision_at_1": 0.45455, "precision_at_3": 0.1929, "precision_at_5": 0.12905, "precision_at_10": 0.07118, "precision_at_20": 0.03825, "precision_at_100": 0.00909, "precision_at_1000": 0.001, "mrr_at_1": 0.45454545454545453, "mrr_at_3": 0.5092387287509238, "mrr_at_5": 0.5245380635624538, "mrr_at_10": 0.5342519269348538, "mrr_at_20": 0.5380380825590302, "mrr_at_100": 0.5419446919495108, "mrr_at_1000": 0.5423785594961397, "naucs_at_1_max": 0.36035015715507857, "naucs_at_1_std": -0.024390145298566285, "naucs_at_1_diff1": 0.7800052609714063, "naucs_at_3_max": 0.36837705741098414, "naucs_at_3_std": -0.020657912235655745, "naucs_at_3_diff1": 0.7084873709787786, "naucs_at_5_max": 0.3875268487915248, "naucs_at_5_std": 0.021337305055989998, "naucs_at_5_diff1": 0.6818968893513976, "naucs_at_10_max": 0.4005361359777307, "naucs_at_10_std": 0.04791798949968782, "naucs_at_10_diff1": 0.6243224308558567, "naucs_at_20_max": 0.38985130710065313, "naucs_at_20_std": 0.07213165772558741, "naucs_at_20_diff1": 0.6186455678853663, "naucs_at_100_max": 0.6399990428160588, "naucs_at_100_std": 0.5164029187350452, "naucs_at_100_diff1": 0.6255315885204668, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "syntheticDocQA_healthcare_industry_test": {"ndcg_at_1": 0.87, "ndcg_at_3": 0.94571, "ndcg_at_5": 0.94958, "ndcg_at_10": 0.94958, "ndcg_at_20": 0.94958, "ndcg_at_100": 0.94958, "ndcg_at_1000": 0.94958, "map_at_1": 0.87, "map_at_3": 0.93, "map_at_5": 0.932, "map_at_10": 0.932, "map_at_20": 0.932, "map_at_100": 0.932, "map_at_1000": 0.932, "recall_at_1": 0.87, "recall_at_3": 0.99, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.87, "precision_at_3": 0.33, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.88, "mrr_at_3": 0.935, "mrr_at_5": 0.937, "mrr_at_10": 0.937, "mrr_at_20": 0.937, "mrr_at_100": 0.937, "mrr_at_1000": 0.937, "naucs_at_1_max": 0.6314892986743688, "naucs_at_1_std": 0.07598311486336452, "naucs_at_1_diff1": 0.8239280159964452, "naucs_at_3_max": 0.8692810457516356, "naucs_at_3_std": 0.12278244631183229, "naucs_at_3_diff1": 0.7222222222222157, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "infovqa_test_subsampled": {"ndcg_at_1": 0.78745, "ndcg_at_3": 0.85213, "ndcg_at_5": 0.86293, "ndcg_at_10": 0.87009, "ndcg_at_20": 0.87574, "ndcg_at_100": 0.87963, "ndcg_at_1000": 0.88086, "map_at_1": 0.78745, "map_at_3": 0.83603, "map_at_5": 0.842, "map_at_10": 0.84493, "map_at_20": 0.84649, "map_at_100": 0.84709, "map_at_1000": 0.84713, "recall_at_1": 0.78745, "recall_at_3": 0.89879, "recall_at_5": 0.9251, "recall_at_10": 0.94737, "recall_at_20": 0.96964, "recall_at_100": 0.98988, "recall_at_1000": 1.0, "precision_at_1": 0.78745, "precision_at_3": 0.2996, "precision_at_5": 0.18502, "precision_at_10": 0.09474, "precision_at_20": 0.04848, "precision_at_100": 0.0099, "precision_at_1000": 0.001, "mrr_at_1": 0.7874493927125507, "mrr_at_3": 0.8363697705802969, "mrr_at_5": 0.8423414304993253, "mrr_at_10": 0.8452670136880663, "mrr_at_20": 0.8469652732557591, "mrr_at_100": 0.8474609459209939, "mrr_at_1000": 0.8474990786545865, "naucs_at_1_max": 0.46539854549496334, "naucs_at_1_std": -0.13294442635379716, "naucs_at_1_diff1": 0.8767412852615252, "naucs_at_3_max": 0.5214681050998872, "naucs_at_3_std": -0.06847991035457092, "naucs_at_3_diff1": 0.813144518489895, "naucs_at_5_max": 0.668769270455956, "naucs_at_5_std": 0.12917742610971458, "naucs_at_5_diff1": 0.8037795219141125, "naucs_at_10_max": 0.6542702167981308, "naucs_at_10_std": 0.2493765160411771, "naucs_at_10_diff1": 0.7869140700302234, "naucs_at_20_max": 0.8701034580540723, "naucs_at_20_std": 0.735101976401789, "naucs_at_20_diff1": 0.7674216348183744, "naucs_at_100_max": 0.892212470822504, "naucs_at_100_std": 0.8586184551815492, "naucs_at_100_diff1": 0.8193497747364105, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "syntheticDocQA_energy_test": {"ndcg_at_1": 0.85, "ndcg_at_3": 0.88524, "ndcg_at_5": 0.90203, "ndcg_at_10": 0.90892, "ndcg_at_20": 0.91683, "ndcg_at_100": 0.91683, "ndcg_at_1000": 0.91683, "map_at_1": 0.85, "map_at_3": 0.87667, "map_at_5": 0.88617, "map_at_10": 0.88926, "map_at_20": 0.8916, "map_at_100": 0.8916, "map_at_1000": 0.8916, "recall_at_1": 0.85, "recall_at_3": 0.91, "recall_at_5": 0.95, "recall_at_10": 0.97, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.85, "precision_at_3": 0.30333, "precision_at_5": 0.19, "precision_at_10": 0.097, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.85, "mrr_at_3": 0.8766666666666667, "mrr_at_5": 0.8861666666666665, "mrr_at_10": 0.8892619047619047, "mrr_at_20": 0.8916428571428571, "mrr_at_100": 0.8916428571428571, "mrr_at_1000": 0.8916428571428571, "naucs_at_1_max": 0.5672958021477384, "naucs_at_1_std": -0.19749430523917919, "naucs_at_1_diff1": 0.8854539537910836, "naucs_at_3_max": 0.508818342151675, "naucs_at_3_std": -0.42841581076875246, "naucs_at_3_diff1": 0.8397136632430731, "naucs_at_5_max": 0.286741363211954, "naucs_at_5_std": -0.49682539682538396, "naucs_at_5_diff1": 0.9477124183006543, "naucs_at_10_max": 0.02521008403361212, "naucs_at_10_std": -0.8249299719887969, "naucs_at_10_diff1": 0.9564270152505424, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "tabfquad_test_subsampled": {"ndcg_at_1": 0.88571, "ndcg_at_3": 0.93155, "ndcg_at_5": 0.93462, "ndcg_at_10": 0.94187, "ndcg_at_20": 0.9428, "ndcg_at_100": 0.94342, "ndcg_at_1000": 0.94342, "map_at_1": 0.88571, "map_at_3": 0.92024, "map_at_5": 0.92202, "map_at_10": 0.92519, "map_at_20": 0.92547, "map_at_100": 0.92553, "map_at_1000": 0.92553, "recall_at_1": 0.88571, "recall_at_3": 0.96429, "recall_at_5": 0.97143, "recall_at_10": 0.99286, "recall_at_20": 0.99643, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.88571, "precision_at_3": 0.32143, "precision_at_5": 0.19429, "precision_at_10": 0.09929, "precision_at_20": 0.04982, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.8857142857142857, "mrr_at_3": 0.919047619047619, "mrr_at_5": 0.9217261904761904, "mrr_at_10": 0.9248937074829932, "mrr_at_20": 0.9251913265306123, "mrr_at_100": 0.9252574640967497, "mrr_at_1000": 0.9252574640967497, "naucs_at_1_max": 0.75249169435216, "naucs_at_1_std": -0.09047223540578946, "naucs_at_1_diff1": 0.8852930707166594, "naucs_at_3_max": 0.9869281045751616, "naucs_at_3_std": -0.15000000000000438, "naucs_at_3_diff1": 0.9161998132586365, "naucs_at_5_max": 0.9836601307189566, "naucs_at_5_std": -0.09564659197011154, "naucs_at_5_diff1": 0.9279295051353875, "naucs_at_10_max": 1.0, "naucs_at_10_std": 0.4960317460317667, "naucs_at_10_diff1": 0.777077497665725, "naucs_at_20_max": 1.0, "naucs_at_20_std": 0.8692810457516478, "naucs_at_20_diff1": 0.5541549953314753, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}} |