{ "vidore/arxivqa_test_subsampled": { "ndcg_at_1": 0.87, "ndcg_at_3": 0.89971, "ndcg_at_5": 0.9146, "ndcg_at_10": 0.92076, "ndcg_at_20": 0.9232, "ndcg_at_50": 0.92445, "ndcg_at_100": 0.92575, "map_at_1": 0.87, "map_at_3": 0.89267, "map_at_5": 0.90097, "map_at_10": 0.9037, "map_at_20": 0.90433, "map_at_50": 0.90455, "map_at_100": 0.90467, "recall_at_1": 0.87, "recall_at_3": 0.92, "recall_at_5": 0.956, "recall_at_10": 0.974, "recall_at_20": 0.984, "recall_at_50": 0.99, "recall_at_100": 0.998, "precision_at_1": 0.87, "precision_at_3": 0.30667, "precision_at_5": 0.1912, "precision_at_10": 0.0974, "precision_at_20": 0.0492, "precision_at_50": 0.0198, "precision_at_100": 0.00998, "mrr_at_1": 0.868, "mrr_at_3": 0.8926666666666665, "mrr_at_5": 0.9010666666666663, "mrr_at_10": 0.9032333333333331, "mrr_at_20": 0.9038726836888601, "mrr_at_50": 0.9040981863343626, "mrr_at_100": 0.9042154113217136, "naucs_at_1_max": 0.7759016514848549, "naucs_at_1_std": -0.07636080870917564, "naucs_at_1_diff1": 0.9178701029400873, "naucs_at_3_max": 0.7505018674136326, "naucs_at_3_std": -0.0976540616246483, "naucs_at_3_diff1": 0.9075630252100865, "naucs_at_5_max": 0.7691409897292251, "naucs_at_5_std": -0.17152618623206986, "naucs_at_5_diff1": 0.913844325609031, "naucs_at_10_max": 0.8861595920419414, "naucs_at_10_std": -0.07297277885513653, "naucs_at_10_diff1": 0.9484665661136262, "naucs_at_20_max": 0.9115896358543447, "naucs_at_20_std": 0.19193510737628516, "naucs_at_20_diff1": 0.9325980392156931, "naucs_at_50_max": 0.947712418300658, "naucs_at_50_std": 0.010924369747883845, "naucs_at_50_diff1": 0.9738562091503188, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0 }, "vidore/docvqa_test_subsampled": { "ndcg_at_1": 0.55654, "ndcg_at_3": 0.62285, "ndcg_at_5": 0.64298, "ndcg_at_10": 0.65674, "ndcg_at_20": 0.6708, "ndcg_at_50": 0.68247, "ndcg_at_100": 0.69013, "map_at_1": 0.55654, "map_at_3": 0.60532, "map_at_5": 0.61652, "map_at_10": 0.62227, "map_at_20": 0.62616, "map_at_50": 0.62813, "map_at_100": 0.62883, "recall_at_1": 0.55654, "recall_at_3": 0.67406, "recall_at_5": 0.72284, "recall_at_10": 0.76497, "recall_at_20": 0.8204, "recall_at_50": 0.87805, "recall_at_100": 0.92461, "precision_at_1": 0.55654, "precision_at_3": 0.22469, "precision_at_5": 0.14457, "precision_at_10": 0.0765, "precision_at_20": 0.04102, "precision_at_50": 0.01756, "precision_at_100": 0.00925, "mrr_at_1": 0.5543237250554324, "mrr_at_3": 0.6056910569105692, "mrr_at_5": 0.6163340724316333, "mrr_at_10": 0.6222565375004397, "mrr_at_20": 0.6263930444590208, "mrr_at_50": 0.6281533502118082, "mrr_at_100": 0.6288252316605568, "naucs_at_1_max": 0.497847351531617, "naucs_at_1_std": -0.33814500917634066, "naucs_at_1_diff1": 0.8645836037088763, "naucs_at_3_max": 0.4701480123071679, "naucs_at_3_std": -0.34741402449178754, "naucs_at_3_diff1": 0.816322894093004, "naucs_at_5_max": 0.4347871177259376, "naucs_at_5_std": -0.3602860416064191, "naucs_at_5_diff1": 0.7758571855344005, "naucs_at_10_max": 0.45324296482414395, "naucs_at_10_std": -0.34596987897850606, "naucs_at_10_diff1": 0.7544028038035664, "naucs_at_20_max": 0.4086941718861015, "naucs_at_20_std": -0.24939767166193863, "naucs_at_20_diff1": 0.7174109964559481, "naucs_at_50_max": 0.4052361174628584, "naucs_at_50_std": -0.12921981686486492, "naucs_at_50_diff1": 0.7068264858128372, "naucs_at_100_max": 0.4303274261407769, "naucs_at_100_std": 0.1893856757251087, "naucs_at_100_diff1": 0.7510538837895515 }, "vidore/infovqa_test_subsampled": { "ndcg_at_1": 0.88664, "ndcg_at_3": 0.91724, "ndcg_at_5": 0.92561, "ndcg_at_10": 0.92884, "ndcg_at_20": 0.93091, "ndcg_at_50": 0.93251, "ndcg_at_100": 0.93353, "map_at_1": 0.88664, "map_at_3": 0.91026, "map_at_5": 0.91491, "map_at_10": 0.91622, "map_at_20": 0.9168, "map_at_50": 0.91706, "map_at_100": 0.91715, "recall_at_1": 0.88664, "recall_at_3": 0.93725, "recall_at_5": 0.95749, "recall_at_10": 0.96761, "recall_at_20": 0.97571, "recall_at_50": 0.98381, "recall_at_100": 0.98988, "precision_at_1": 0.88664, "precision_at_3": 0.31242, "precision_at_5": 0.1915, "precision_at_10": 0.09676, "precision_at_20": 0.04879, "precision_at_50": 0.01968, "precision_at_100": 0.0099, "mrr_at_1": 0.8846153846153846, "mrr_at_3": 0.9095816464237515, "mrr_at_5": 0.9137314439946017, "mrr_at_10": 0.9150022492127753, "mrr_at_20": 0.9156172230591575, "mrr_at_50": 0.9158740133336603, "mrr_at_100": 0.9159712069854167, "naucs_at_1_max": 0.6570420185225749, "naucs_at_1_std": 0.021742422508079725, "naucs_at_1_diff1": 0.9406199945564582, "naucs_at_3_max": 0.6318777024153259, "naucs_at_3_std": -0.13867851955525243, "naucs_at_3_diff1": 0.9166153687134068, "naucs_at_5_max": 0.7688027815053647, "naucs_at_5_std": 0.12215100872667127, "naucs_at_5_diff1": 0.908003985735216, "naucs_at_10_max": 0.7611694369997123, "naucs_at_10_std": 0.2568374691733195, "naucs_at_10_diff1": 0.8966136523873394, "naucs_at_20_max": 0.9302075684869188, "naucs_at_20_std": 0.5376863638154207, "naucs_at_20_diff1": 0.9224381659931805, "naucs_at_50_max": 0.9116365344884687, "naucs_at_50_std": 0.5788807451893666, "naucs_at_50_diff1": 0.9346992729676175, "naucs_at_100_max": 0.8847387459944923, "naucs_at_100_std": 0.5437597991332108, "naucs_at_100_diff1": 0.9477594183740937 }, "vidore/tabfquad_test_subsampled": { "ndcg_at_1": 0.93571, "ndcg_at_3": 0.95459, "ndcg_at_5": 0.96059, "ndcg_at_10": 0.96527, "ndcg_at_20": 0.96527, "ndcg_at_50": 0.96527, "ndcg_at_100": 0.96589, "map_at_1": 0.93571, "map_at_3": 0.95, "map_at_5": 0.95339, "map_at_10": 0.95537, "map_at_20": 0.95537, "map_at_50": 0.95537, "map_at_100": 0.95543, "recall_at_1": 0.93571, "recall_at_3": 0.96786, "recall_at_5": 0.98214, "recall_at_10": 0.99643, "recall_at_20": 0.99643, "recall_at_50": 0.99643, "recall_at_100": 1.0, "precision_at_1": 0.93571, "precision_at_3": 0.32262, "precision_at_5": 0.19643, "precision_at_10": 0.09964, "precision_at_20": 0.04982, "precision_at_50": 0.01993, "precision_at_100": 0.01, "mrr_at_1": 0.9357142857142857, "mrr_at_3": 0.9511904761904761, "mrr_at_5": 0.9535119047619046, "mrr_at_10": 0.955484693877551, "mrr_at_20": 0.955484693877551, "mrr_at_50": 0.955484693877551, "mrr_at_100": 0.955548469387755, "naucs_at_1_max": 0.5400197115883385, "naucs_at_1_std": -0.014264965245358233, "naucs_at_1_diff1": 0.9125946674966273, "naucs_at_3_max": 0.71428571428571, "naucs_at_3_std": 0.3626932254383213, "naucs_at_3_diff1": 0.9110384894698589, "naucs_at_5_max": 0.8245564892623743, "naucs_at_5_std": 0.3671335200747002, "naucs_at_5_diff1": 0.9183006535947714, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 0.8692810457516478, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 0.8692810457516478, "naucs_at_50_max": 1.0, "naucs_at_50_std": 1.0, "naucs_at_50_diff1": 0.8692810457515607, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0 }, "vidore/tatdqa_test": { "ndcg_at_1": 0.66464, "ndcg_at_3": 0.76743, "ndcg_at_5": 0.78563, "ndcg_at_10": 0.80598, "ndcg_at_20": 0.81164, "ndcg_at_50": 0.8141, "ndcg_at_100": 0.81571, "map_at_1": 0.66464, "map_at_3": 0.74281, "map_at_5": 0.75302, "map_at_10": 0.7616, "map_at_20": 0.76322, "map_at_50": 0.76363, "map_at_100": 0.76379, "recall_at_1": 0.66464, "recall_at_3": 0.8384, "recall_at_5": 0.88214, "recall_at_10": 0.94411, "recall_at_20": 0.96598, "recall_at_50": 0.97813, "recall_at_100": 0.98785, "precision_at_1": 0.66464, "precision_at_3": 0.27947, "precision_at_5": 0.17643, "precision_at_10": 0.09441, "precision_at_20": 0.0483, "precision_at_50": 0.01956, "precision_at_100": 0.00988, "mrr_at_1": 0.6682867557715675, "mrr_at_3": 0.7444309437019037, "mrr_at_5": 0.7550931551235324, "mrr_at_10": 0.7633802001967253, "mrr_at_20": 0.7649268100771701, "mrr_at_50": 0.7653640518991484, "mrr_at_100": 0.7655149926790836, "naucs_at_1_max": 0.1850885816800905, "naucs_at_1_std": -0.28841580839611636, "naucs_at_1_diff1": 0.8092207194564194, "naucs_at_3_max": 0.22777444593854884, "naucs_at_3_std": -0.19766587895858276, "naucs_at_3_diff1": 0.6835545302979953, "naucs_at_5_max": 0.24718915870776953, "naucs_at_5_std": -0.20386104103344074, "naucs_at_5_diff1": 0.6717631348394746, "naucs_at_10_max": 0.4129645887196616, "naucs_at_10_std": 0.08874265042732231, "naucs_at_10_diff1": 0.6599246442060299, "naucs_at_20_max": 0.36324695580118865, "naucs_at_20_std": 0.25905536406088486, "naucs_at_20_diff1": 0.661930000003523, "naucs_at_50_max": 0.4870532250141124, "naucs_at_50_std": 0.42734413587689424, "naucs_at_50_diff1": 0.6936991113088404, "naucs_at_100_max": 0.6138710077391127, "naucs_at_100_std": 0.610321256875322, "naucs_at_100_diff1": 0.6717110238397306 }, "vidore/shiftproject_test": { "ndcg_at_1": 0.75, "ndcg_at_3": 0.86702, "ndcg_at_5": 0.87906, "ndcg_at_10": 0.88207, "ndcg_at_20": 0.88207, "ndcg_at_50": 0.88207, "ndcg_at_100": 0.88362, "map_at_1": 0.75, "map_at_3": 0.83833, "map_at_5": 0.84483, "map_at_10": 0.84594, "map_at_20": 0.84594, "map_at_50": 0.84594, "map_at_100": 0.84606, "recall_at_1": 0.75, "recall_at_3": 0.95, "recall_at_5": 0.98, "recall_at_10": 0.99, "recall_at_20": 0.99, "recall_at_50": 0.99, "recall_at_100": 1.0, "precision_at_1": 0.75, "precision_at_3": 0.31667, "precision_at_5": 0.196, "precision_at_10": 0.099, "precision_at_20": 0.0495, "precision_at_50": 0.0198, "precision_at_100": 0.01, "mrr_at_1": 0.77, "mrr_at_3": 0.8516666666666667, "mrr_at_5": 0.8586666666666666, "mrr_at_10": 0.8597777777777778, "mrr_at_20": 0.8597777777777778, "mrr_at_50": 0.8597777777777778, "mrr_at_100": 0.8598954248366013, "naucs_at_1_max": 0.1504761904761907, "naucs_at_1_std": -0.2593406593406595, "naucs_at_1_diff1": 0.84887493458922, "naucs_at_3_max": 0.4772175536881391, "naucs_at_3_std": 0.00018674136321205462, "naucs_at_3_diff1": 0.7637721755368763, "naucs_at_5_max": 0.540149393090577, "naucs_at_5_std": 0.1914098972922579, "naucs_at_5_diff1": 0.9346405228758136, "naucs_at_10_max": 0.7222222222222276, "naucs_at_10_std": 0.5541549953314738, "naucs_at_10_diff1": 0.8692810457516413, "naucs_at_20_max": 0.7222222222222276, "naucs_at_20_std": 0.5541549953314738, "naucs_at_20_diff1": 0.8692810457516413, "naucs_at_50_max": 0.7222222222222041, "naucs_at_50_std": 0.554154995331464, "naucs_at_50_diff1": 0.8692810457516374, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_artificial_intelligence_test": { "ndcg_at_1": 0.99, "ndcg_at_3": 0.99631, "ndcg_at_5": 0.99631, "ndcg_at_10": 0.99631, "ndcg_at_20": 0.99631, "ndcg_at_50": 0.99631, "ndcg_at_100": 0.99631, "map_at_1": 0.99, "map_at_3": 0.995, "map_at_5": 0.995, "map_at_10": 0.995, "map_at_20": 0.995, "map_at_50": 0.995, "map_at_100": 0.995, "recall_at_1": 0.99, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.99, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.99, "mrr_at_3": 0.995, "mrr_at_5": 0.995, "mrr_at_10": 0.995, "mrr_at_20": 0.995, "mrr_at_50": 0.995, "mrr_at_100": 0.995, "naucs_at_1_max": 0.12278244631185359, "naucs_at_1_std": 0.12278244631185359, "naucs_at_1_diff1": 1.0, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_energy_test": { "ndcg_at_1": 0.93, "ndcg_at_3": 0.94893, "ndcg_at_5": 0.95323, "ndcg_at_10": 0.95972, "ndcg_at_20": 0.95972, "ndcg_at_50": 0.96159, "ndcg_at_100": 0.96159, "map_at_1": 0.93, "map_at_3": 0.945, "map_at_5": 0.9475, "map_at_10": 0.95018, "map_at_20": 0.95018, "map_at_50": 0.95043, "map_at_100": 0.95043, "recall_at_1": 0.93, "recall_at_3": 0.96, "recall_at_5": 0.97, "recall_at_10": 0.99, "recall_at_20": 0.99, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.93, "precision_at_3": 0.32, "precision_at_5": 0.194, "precision_at_10": 0.099, "precision_at_20": 0.0495, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.94, "mrr_at_3": 0.95, "mrr_at_5": 0.9545, "mrr_at_10": 0.9561666666666667, "mrr_at_20": 0.9561666666666667, "mrr_at_50": 0.9564298245614035, "mrr_at_100": 0.9564298245614035, "naucs_at_1_max": 0.30052020808323304, "naucs_at_1_std": -0.6699346405228773, "naucs_at_1_diff1": 0.9813258636788056, "naucs_at_3_max": 0.7864145658263298, "naucs_at_3_std": -0.6272175536881414, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 0.7587924058512269, "naucs_at_5_std": -1.0210084033613454, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 0.7222222222222276, "naucs_at_10_std": -1.1517273576097316, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 0.7222222222222276, "naucs_at_20_std": -1.1517273576097316, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_government_reports_test": { "ndcg_at_1": 0.91, "ndcg_at_3": 0.94655, "ndcg_at_5": 0.95947, "ndcg_at_10": 0.95947, "ndcg_at_20": 0.95947, "ndcg_at_50": 0.95947, "ndcg_at_100": 0.95947, "map_at_1": 0.91, "map_at_3": 0.93833, "map_at_5": 0.94583, "map_at_10": 0.94583, "map_at_20": 0.94583, "map_at_50": 0.94583, "map_at_100": 0.94583, "recall_at_1": 0.91, "recall_at_3": 0.97, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.91, "precision_at_3": 0.32333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.93, "mrr_at_3": 0.955, "mrr_at_5": 0.9575, "mrr_at_10": 0.9575, "mrr_at_20": 0.9575, "mrr_at_50": 0.9575, "mrr_at_100": 0.9575, "naucs_at_1_max": 0.6442577030812321, "naucs_at_1_std": 0.19950202303143538, "naucs_at_1_diff1": 0.8887332710862121, "naucs_at_3_max": 0.807812013694371, "naucs_at_3_std": 0.3029878618113896, "naucs_at_3_diff1": 0.9074074074074082, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_healthcare_industry_test": { "ndcg_at_1": 0.99, "ndcg_at_3": 0.99631, "ndcg_at_5": 0.99631, "ndcg_at_10": 0.99631, "ndcg_at_20": 0.99631, "ndcg_at_50": 0.99631, "ndcg_at_100": 0.99631, "map_at_1": 0.99, "map_at_3": 0.995, "map_at_5": 0.995, "map_at_10": 0.995, "map_at_20": 0.995, "map_at_50": 0.995, "map_at_100": 0.995, "recall_at_1": 0.99, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.99, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.99, "mrr_at_3": 0.995, "mrr_at_5": 0.995, "mrr_at_10": 0.995, "mrr_at_20": 0.995, "mrr_at_50": 0.995, "mrr_at_100": 0.995, "naucs_at_1_max": 0.554154995331463, "naucs_at_1_std": 0.7222222222222201, "naucs_at_1_diff1": 0.8692810457516276, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null } }