{ "vidore/arxivqa_test_subsampled": { "ndcg_at_1": 0.852, "ndcg_at_3": 0.90086, "ndcg_at_5": 0.90722, "ndcg_at_10": 0.91102, "ndcg_at_20": 0.91656, "ndcg_at_50": 0.91851, "ndcg_at_100": 0.91914, "map_at_1": 0.852, "map_at_3": 0.88933, "map_at_5": 0.89273, "map_at_10": 0.89425, "map_at_20": 0.89575, "map_at_50": 0.89605, "map_at_100": 0.89611, "recall_at_1": 0.852, "recall_at_3": 0.934, "recall_at_5": 0.95, "recall_at_10": 0.962, "recall_at_20": 0.984, "recall_at_50": 0.994, "recall_at_100": 0.998, "precision_at_1": 0.852, "precision_at_3": 0.31133, "precision_at_5": 0.19, "precision_at_10": 0.0962, "precision_at_20": 0.0492, "precision_at_50": 0.01988, "precision_at_100": 0.00998, "mrr_at_1": 0.852, "mrr_at_3": 0.8879999999999998, "mrr_at_5": 0.8918999999999998, "mrr_at_10": 0.8935880952380951, "mrr_at_20": 0.8949619694031458, "mrr_at_50": 0.8952206190465779, "mrr_at_100": 0.8953101826793595, "naucs_at_1_max": 0.743886209713547, "naucs_at_1_std": -0.16796452947532062, "naucs_at_1_diff1": 0.9333280628244656, "naucs_at_3_max": 0.7297060238236761, "naucs_at_3_std": -0.29585773703420537, "naucs_at_3_diff1": 0.916532269473447, "naucs_at_5_max": 0.7750140056022408, "naucs_at_5_std": -0.3126797385620943, "naucs_at_5_diff1": 0.9292250233426705, "naucs_at_10_max": 0.7392255147673084, "naucs_at_10_std": -0.24276377217554787, "naucs_at_10_diff1": 0.9137549756744823, "naucs_at_20_max": 0.9115896358543447, "naucs_at_20_std": 0.07142857142856779, "naucs_at_20_diff1": 0.9836601307189491, "naucs_at_50_max": 0.9564270152505304, "naucs_at_50_std": 0.08667911609088286, "naucs_at_50_diff1": 1.0, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0 }, "vidore/docvqa_test_subsampled": { "ndcg_at_1": 0.55654, "ndcg_at_3": 0.61596, "ndcg_at_5": 0.64525, "ndcg_at_10": 0.65672, "ndcg_at_20": 0.67079, "ndcg_at_50": 0.68331, "ndcg_at_100": 0.69204, "map_at_1": 0.55654, "map_at_3": 0.60126, "map_at_5": 0.61755, "map_at_10": 0.62229, "map_at_20": 0.62618, "map_at_50": 0.62808, "map_at_100": 0.62888, "recall_at_1": 0.55654, "recall_at_3": 0.65854, "recall_at_5": 0.72949, "recall_at_10": 0.76497, "recall_at_20": 0.8204, "recall_at_50": 0.8847, "recall_at_100": 0.93792, "precision_at_1": 0.55654, "precision_at_3": 0.21951, "precision_at_5": 0.1459, "precision_at_10": 0.0765, "precision_at_20": 0.04102, "precision_at_50": 0.01769, "precision_at_100": 0.00938, "mrr_at_1": 0.5476718403547672, "mrr_at_3": 0.598669623059867, "mrr_at_5": 0.6114190687361419, "mrr_at_10": 0.6178043501214233, "mrr_at_20": 0.621799912006316, "mrr_at_50": 0.6237201129227461, "mrr_at_100": 0.6245233026011214, "naucs_at_1_max": 0.4832573617906144, "naucs_at_1_std": -0.3032135796994576, "naucs_at_1_diff1": 0.85261473666465, "naucs_at_3_max": 0.46413211861079323, "naucs_at_3_std": -0.3586604308785551, "naucs_at_3_diff1": 0.8210611793068221, "naucs_at_5_max": 0.448455784463693, "naucs_at_5_std": -0.3356088095049938, "naucs_at_5_diff1": 0.7876237365986988, "naucs_at_10_max": 0.4180379024328761, "naucs_at_10_std": -0.2978414780805468, "naucs_at_10_diff1": 0.7518139587785174, "naucs_at_20_max": 0.3539033589636336, "naucs_at_20_std": -0.29617967205487483, "naucs_at_20_diff1": 0.7229362478946776, "naucs_at_50_max": 0.3294934197988531, "naucs_at_50_std": -0.026930976978897465, "naucs_at_50_diff1": 0.6978098616006386, "naucs_at_100_max": 0.4056438414696139, "naucs_at_100_std": 0.37753201203052666, "naucs_at_100_diff1": 0.6820256833963592 }, "vidore/infovqa_test_subsampled": { "ndcg_at_1": 0.88057, "ndcg_at_3": 0.90936, "ndcg_at_5": 0.92034, "ndcg_at_10": 0.92497, "ndcg_at_20": 0.92755, "ndcg_at_50": 0.92954, "ndcg_at_100": 0.93021, "map_at_1": 0.88057, "map_at_3": 0.9025, "map_at_5": 0.90867, "map_at_10": 0.9106, "map_at_20": 0.91132, "map_at_50": 0.91163, "map_at_100": 0.9117, "recall_at_1": 0.88057, "recall_at_3": 0.92915, "recall_at_5": 0.95547, "recall_at_10": 0.96964, "recall_at_20": 0.97976, "recall_at_50": 0.98988, "recall_at_100": 0.99393, "precision_at_1": 0.88057, "precision_at_3": 0.30972, "precision_at_5": 0.19109, "precision_at_10": 0.09696, "precision_at_20": 0.04899, "precision_at_50": 0.0198, "precision_at_100": 0.00994, "mrr_at_1": 0.8785425101214575, "mrr_at_3": 0.9011470985155193, "mrr_at_5": 0.9075236167341428, "mrr_at_10": 0.9092547072810231, "mrr_at_20": 0.9101327765801449, "mrr_at_50": 0.9104553627424199, "mrr_at_100": 0.9105182434033097, "naucs_at_1_max": 0.6442747044878268, "naucs_at_1_std": -0.019608341053496993, "naucs_at_1_diff1": 0.9484048829634778, "naucs_at_3_max": 0.7386798922430662, "naucs_at_3_std": 0.03322023115905618, "naucs_at_3_diff1": 0.9116914983852844, "naucs_at_5_max": 0.772590092296685, "naucs_at_5_std": 0.015327130682268415, "naucs_at_5_diff1": 0.9136885163498384, "naucs_at_10_max": 0.8058420528202048, "naucs_at_10_std": 0.12522140546049776, "naucs_at_10_diff1": 0.9267525275808962, "naucs_at_20_max": 0.8138015202314234, "naucs_at_20_std": 0.3863206159107717, "naucs_at_20_diff1": 0.9162490821843028, "naucs_at_50_max": 0.8291917984429228, "naucs_at_50_std": 0.4153501554955276, "naucs_at_50_diff1": 0.9477594183740937, "naucs_at_100_max": 0.7588534820930969, "naucs_at_100_std": 0.2395996652220107, "naucs_at_100_diff1": 0.9564661819784259 }, "vidore/tabfquad_test_subsampled": { "ndcg_at_1": 0.91786, "ndcg_at_3": 0.94753, "ndcg_at_5": 0.95045, "ndcg_at_10": 0.95857, "ndcg_at_20": 0.95857, "ndcg_at_50": 0.95857, "ndcg_at_100": 0.95857, "map_at_1": 0.91786, "map_at_3": 0.94048, "map_at_5": 0.94208, "map_at_10": 0.94545, "map_at_20": 0.94545, "map_at_50": 0.94545, "map_at_100": 0.94545, "recall_at_1": 0.91786, "recall_at_3": 0.96786, "recall_at_5": 0.975, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.91786, "precision_at_3": 0.32262, "precision_at_5": 0.195, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.9214285714285714, "mrr_at_3": 0.9422619047619047, "mrr_at_5": 0.9438690476190476, "mrr_at_10": 0.9472406462585033, "mrr_at_20": 0.9472406462585033, "mrr_at_50": 0.9472406462585033, "mrr_at_100": 0.9472406462585033, "naucs_at_1_max": 0.5557179393496541, "naucs_at_1_std": 0.021881216254618023, "naucs_at_1_diff1": 0.937989688629075, "naucs_at_3_max": 0.8384687208216648, "naucs_at_3_std": 0.5060690943043917, "naucs_at_3_diff1": 0.9709513435003594, "naucs_at_5_max": 0.9176337201547285, "naucs_at_5_std": 0.5883019874616514, "naucs_at_5_diff1": 0.9626517273576021, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": 1.0, "naucs_at_50_std": 1.0, "naucs_at_50_diff1": 1.0, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0 }, "vidore/tatdqa_test": { "ndcg_at_1": 0.66707, "ndcg_at_3": 0.76435, "ndcg_at_5": 0.78563, "ndcg_at_10": 0.80266, "ndcg_at_20": 0.81002, "ndcg_at_50": 0.81426, "ndcg_at_100": 0.81556, "map_at_1": 0.66707, "map_at_3": 0.74099, "map_at_5": 0.75296, "map_at_10": 0.76017, "map_at_20": 0.76227, "map_at_50": 0.76296, "map_at_100": 0.76308, "recall_at_1": 0.66707, "recall_at_3": 0.83171, "recall_at_5": 0.88275, "recall_at_10": 0.93439, "recall_at_20": 0.96294, "recall_at_50": 0.9842, "recall_at_100": 0.9921, "precision_at_1": 0.66707, "precision_at_3": 0.27724, "precision_at_5": 0.17655, "precision_at_10": 0.09344, "precision_at_20": 0.04815, "precision_at_50": 0.01968, "precision_at_100": 0.00992, "mrr_at_1": 0.6701093560145808, "mrr_at_3": 0.7442284325637911, "mrr_at_5": 0.7551336573511549, "mrr_at_10": 0.7622198595922778, "mrr_at_20": 0.7641436261113056, "mrr_at_50": 0.7648243083644104, "mrr_at_100": 0.764956538385537, "naucs_at_1_max": 0.16664544436515671, "naucs_at_1_std": -0.3026655256139227, "naucs_at_1_diff1": 0.8139267777793499, "naucs_at_3_max": 0.2218720138316067, "naucs_at_3_std": -0.2588980164852128, "naucs_at_3_diff1": 0.7085637741457546, "naucs_at_5_max": 0.21174343657776057, "naucs_at_5_std": -0.223401297475484, "naucs_at_5_diff1": 0.6871017298675983, "naucs_at_10_max": 0.34063196744044216, "naucs_at_10_std": -0.08738627538479347, "naucs_at_10_diff1": 0.7014648602426637, "naucs_at_20_max": 0.4795886093835319, "naucs_at_20_std": 0.18366030249088966, "naucs_at_20_diff1": 0.656419869346376, "naucs_at_50_max": 0.6537769988723047, "naucs_at_50_std": 0.5912022233191522, "naucs_at_50_diff1": 0.6681893202319851, "naucs_at_100_max": 0.5982571033156504, "naucs_at_100_std": 0.5589672729076165, "naucs_at_100_diff1": 0.6339270114131129 }, "vidore/shiftproject_test": { "ndcg_at_1": 0.8, "ndcg_at_3": 0.8844, "ndcg_at_5": 0.89258, "ndcg_at_10": 0.90263, "ndcg_at_20": 0.90263, "ndcg_at_50": 0.90446, "ndcg_at_100": 0.90446, "map_at_1": 0.8, "map_at_3": 0.865, "map_at_5": 0.8695, "map_at_10": 0.87385, "map_at_20": 0.87385, "map_at_50": 0.87408, "map_at_100": 0.87408, "recall_at_1": 0.8, "recall_at_3": 0.94, "recall_at_5": 0.96, "recall_at_10": 0.99, "recall_at_20": 0.99, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.8, "precision_at_3": 0.31333, "precision_at_5": 0.192, "precision_at_10": 0.099, "precision_at_20": 0.0495, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.84, "mrr_at_3": 0.8883333333333334, "mrr_at_5": 0.8933333333333334, "mrr_at_10": 0.8976785714285715, "mrr_at_20": 0.8976785714285715, "mrr_at_50": 0.8979111295681064, "mrr_at_100": 0.8979111295681064, "naucs_at_1_max": -0.04510489510489574, "naucs_at_1_std": -0.40072427572427605, "naucs_at_1_diff1": 0.7706293706293702, "naucs_at_3_max": 0.41651104886398477, "naucs_at_3_std": 0.053299097416741235, "naucs_at_3_diff1": 0.679816370992838, "naucs_at_5_max": 0.6626984126984157, "naucs_at_5_std": 0.1178804855275477, "naucs_at_5_diff1": 0.7916666666666669, "naucs_at_10_max": 0.5541549953314738, "naucs_at_10_std": 0.35807656395891135, "naucs_at_10_diff1": 0.7222222222222276, "naucs_at_20_max": 0.5541549953314738, "naucs_at_20_std": 0.35807656395891135, "naucs_at_20_diff1": 0.7222222222222276, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_artificial_intelligence_test": { "ndcg_at_1": 0.98, "ndcg_at_3": 0.99262, "ndcg_at_5": 0.99262, "ndcg_at_10": 0.99262, "ndcg_at_20": 0.99262, "ndcg_at_50": 0.99262, "ndcg_at_100": 0.99262, "map_at_1": 0.98, "map_at_3": 0.99, "map_at_5": 0.99, "map_at_10": 0.99, "map_at_20": 0.99, "map_at_50": 0.99, "map_at_100": 0.99, "recall_at_1": 0.98, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.98, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.98, "mrr_at_3": 0.99, "mrr_at_5": 0.99, "mrr_at_10": 0.99, "mrr_at_20": 0.99, "mrr_at_50": 0.99, "mrr_at_100": 0.99, "naucs_at_1_max": 0.4561157796451899, "naucs_at_1_std": 0.4225023342670396, "naucs_at_1_diff1": 1.0, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_energy_test": { "ndcg_at_1": 0.95, "ndcg_at_3": 0.96262, "ndcg_at_5": 0.96262, "ndcg_at_10": 0.96951, "ndcg_at_20": 0.96951, "ndcg_at_50": 0.97137, "ndcg_at_100": 0.97137, "map_at_1": 0.95, "map_at_3": 0.96, "map_at_5": 0.96, "map_at_10": 0.9631, "map_at_20": 0.9631, "map_at_50": 0.96334, "map_at_100": 0.96334, "recall_at_1": 0.95, "recall_at_3": 0.97, "recall_at_5": 0.97, "recall_at_10": 0.99, "recall_at_20": 0.99, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.95, "precision_at_3": 0.32333, "precision_at_5": 0.194, "precision_at_10": 0.099, "precision_at_20": 0.0495, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.95, "mrr_at_3": 0.96, "mrr_at_5": 0.9625, "mrr_at_10": 0.9641666666666667, "mrr_at_20": 0.9641666666666667, "mrr_at_50": 0.9644166666666666, "mrr_at_100": 0.9644166666666666, "naucs_at_1_max": 0.33669467787114943, "naucs_at_1_std": -0.9281979458450001, "naucs_at_1_diff1": 1.0, "naucs_at_3_max": 0.807812013694371, "naucs_at_3_std": -1.1517273576097098, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 0.807812013694364, "naucs_at_5_std": -1.1517273576097071, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": -1.1517273576097316, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": -1.1517273576097316, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_government_reports_test": { "ndcg_at_1": 0.92, "ndcg_at_3": 0.95024, "ndcg_at_5": 0.96316, "ndcg_at_10": 0.96316, "ndcg_at_20": 0.96316, "ndcg_at_50": 0.96316, "ndcg_at_100": 0.96316, "map_at_1": 0.92, "map_at_3": 0.94333, "map_at_5": 0.95083, "map_at_10": 0.95083, "map_at_20": 0.95083, "map_at_50": 0.95083, "map_at_100": 0.95083, "recall_at_1": 0.92, "recall_at_3": 0.97, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.92, "precision_at_3": 0.32333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.93, "mrr_at_3": 0.9516666666666667, "mrr_at_5": 0.9566666666666667, "mrr_at_10": 0.9566666666666667, "mrr_at_20": 0.9566666666666667, "mrr_at_50": 0.9566666666666667, "mrr_at_100": 0.9566666666666667, "naucs_at_1_max": 0.6310107376283841, "naucs_at_1_std": 0.33601774042950283, "naucs_at_1_diff1": 0.9509803921568633, "naucs_at_3_max": 0.7860255213196378, "naucs_at_3_std": 0.24696545284781227, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_healthcare_industry_test": { "ndcg_at_1": 0.98, "ndcg_at_3": 0.99262, "ndcg_at_5": 0.99262, "ndcg_at_10": 0.99262, "ndcg_at_20": 0.99262, "ndcg_at_50": 0.99262, "ndcg_at_100": 0.99262, "map_at_1": 0.98, "map_at_3": 0.99, "map_at_5": 0.99, "map_at_10": 0.99, "map_at_20": 0.99, "map_at_50": 0.99, "map_at_100": 0.99, "recall_at_1": 0.98, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.98, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.98, "mrr_at_3": 0.99, "mrr_at_5": 0.99, "mrr_at_10": 0.99, "mrr_at_20": 0.99, "mrr_at_50": 0.99, "mrr_at_100": 0.99, "naucs_at_1_max": 0.7222222222222248, "naucs_at_1_std": 0.5613912231559305, "naucs_at_1_diff1": 0.9346405228758133, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null } }