{ "metadata": { "timestamp": "2025-06-26T06:21:27.128658", "vidore_benchmark_version": "5.0.1.dev12+ge0eb903" }, "metrics": { "vidore/arxivqa_test_subsampled": { "ndcg_at_1": 0.834, "ndcg_at_3": 0.87602, "ndcg_at_5": 0.88351, "ndcg_at_10": 0.89382, "ndcg_at_20": 0.89856, "ndcg_at_50": 0.9021, "ndcg_at_100": 0.90271, "map_at_1": 0.834, "map_at_3": 0.86567, "map_at_5": 0.86987, "map_at_10": 0.87409, "map_at_20": 0.87549, "map_at_50": 0.87605, "map_at_100": 0.87609, "recall_at_1": 0.834, "recall_at_3": 0.906, "recall_at_5": 0.924, "recall_at_10": 0.956, "recall_at_20": 0.974, "recall_at_50": 0.992, "recall_at_100": 0.996, "precision_at_1": 0.834, "precision_at_3": 0.302, "precision_at_5": 0.1848, "precision_at_10": 0.0956, "precision_at_20": 0.0487, "precision_at_50": 0.01984, "precision_at_100": 0.00996, "mrr_at_1": 0.834, "mrr_at_3": 0.8656666666666666, "mrr_at_5": 0.8698666666666666, "mrr_at_10": 0.8740904761904763, "mrr_at_20": 0.8754917376740906, "mrr_at_50": 0.8760454757022237, "mrr_at_100": 0.8760882870575352, "naucs_at_1_max": 0.6703392545655197, "naucs_at_1_std": 0.4132759009409903, "naucs_at_1_diff1": 0.9520605650682898, "naucs_at_3_max": 0.6840495063273536, "naucs_at_3_std": 0.45158630828217644, "naucs_at_3_diff1": 0.9315116117368949, "naucs_at_5_max": 0.7034989434370238, "naucs_at_5_std": 0.46818025455796014, "naucs_at_5_diff1": 0.9329205366357055, "naucs_at_10_max": 0.6890968508615603, "naucs_at_10_std": 0.474195738901625, "naucs_at_10_diff1": 0.9457813428401673, "naucs_at_20_max": 0.7880126409538182, "naucs_at_20_std": 0.5496301084536358, "naucs_at_20_diff1": 0.9283559577677175, "naucs_at_50_max": 0.7480158730158629, "naucs_at_50_std": 0.8190943043884249, "naucs_at_50_diff1": 0.8978758169934562, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0 }, "vidore/docvqa_test_subsampled": { "ndcg_at_1": 0.58537, "ndcg_at_3": 0.64531, "ndcg_at_5": 0.65942, "ndcg_at_10": 0.67939, "ndcg_at_20": 0.69023, "ndcg_at_50": 0.70398, "ndcg_at_100": 0.71126, "map_at_1": 0.58537, "map_at_3": 0.63008, "map_at_5": 0.63762, "map_at_10": 0.64581, "map_at_20": 0.64889, "map_at_50": 0.65114, "map_at_100": 0.65181, "recall_at_1": 0.58537, "recall_at_3": 0.68958, "recall_at_5": 0.72506, "recall_at_10": 0.78714, "recall_at_20": 0.82927, "recall_at_50": 0.898, "recall_at_100": 0.94235, "precision_at_1": 0.58537, "precision_at_3": 0.22986, "precision_at_5": 0.14501, "precision_at_10": 0.07871, "precision_at_20": 0.04146, "precision_at_50": 0.01796, "precision_at_100": 0.00942, "mrr_at_1": 0.5853658536585366, "mrr_at_3": 0.6300813008130081, "mrr_at_5": 0.6376201034737617, "mrr_at_10": 0.6458064970260089, "mrr_at_20": 0.6488879496173225, "mrr_at_50": 0.6511443628922823, "mrr_at_100": 0.6518089180219389, "naucs_at_1_max": 0.2585563149452475, "naucs_at_1_std": 0.3159595366492725, "naucs_at_1_diff1": 0.8678053985855875, "naucs_at_3_max": 0.22128557788133152, "naucs_at_3_std": 0.258482179935891, "naucs_at_3_diff1": 0.8393573283409745, "naucs_at_5_max": 0.21064270306895383, "naucs_at_5_std": 0.23472397866953829, "naucs_at_5_diff1": 0.8274386494106453, "naucs_at_10_max": 0.15097771381217617, "naucs_at_10_std": 0.2839241224708214, "naucs_at_10_diff1": 0.7893641996302178, "naucs_at_20_max": 0.15466073118900037, "naucs_at_20_std": 0.33553862379172555, "naucs_at_20_diff1": 0.7724781458006538, "naucs_at_50_max": 0.10906657289614331, "naucs_at_50_std": 0.5786081651360842, "naucs_at_50_diff1": 0.7426293121947367, "naucs_at_100_max": 0.072673798370254, "naucs_at_100_std": 0.8933863552254951, "naucs_at_100_diff1": 0.7602442332060952 }, "vidore/infovqa_test_subsampled": { "ndcg_at_1": 0.91498, "ndcg_at_3": 0.94325, "ndcg_at_5": 0.94908, "ndcg_at_10": 0.95095, "ndcg_at_20": 0.95363, "ndcg_at_50": 0.95442, "ndcg_at_100": 0.95476, "map_at_1": 0.91498, "map_at_3": 0.93623, "map_at_5": 0.93947, "map_at_10": 0.94019, "map_at_20": 0.94099, "map_at_50": 0.94111, "map_at_100": 0.94114, "recall_at_1": 0.91498, "recall_at_3": 0.96356, "recall_at_5": 0.97773, "recall_at_10": 0.98381, "recall_at_20": 0.99393, "recall_at_50": 0.99798, "recall_at_100": 1.0, "precision_at_1": 0.91498, "precision_at_3": 0.32119, "precision_at_5": 0.19555, "precision_at_10": 0.09838, "precision_at_20": 0.0497, "precision_at_50": 0.01996, "precision_at_100": 0.01, "mrr_at_1": 0.9149797570850202, "mrr_at_3": 0.936234817813765, "mrr_at_5": 0.9394736842105261, "mrr_at_10": 0.9401902191375874, "mrr_at_20": 0.9409887775689154, "mrr_at_50": 0.9411103076140448, "mrr_at_100": 0.9411405209199847, "naucs_at_1_max": 0.6703820792124877, "naucs_at_1_std": 0.16833937392893533, "naucs_at_1_diff1": 0.9498534501270284, "naucs_at_3_max": 0.5822492726969997, "naucs_at_3_std": 0.06259669593622658, "naucs_at_3_diff1": 0.9492105456414766, "naucs_at_5_max": 0.739556315880861, "naucs_at_5_std": 0.5078895506993929, "naucs_at_5_diff1": 0.9643814216187027, "naucs_at_10_max": 0.6766067765559199, "naucs_at_10_std": 0.3559984957278538, "naucs_at_10_diff1": 0.9836748182418953, "naucs_at_20_max": 0.9564661819784134, "naucs_at_20_std": 0.8638879360590604, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": 0.8693985459351681, "naucs_at_50_std": 0.8693985459351681, "naucs_at_50_diff1": 1.0, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/tabfquad_test_subsampled": { "ndcg_at_1": 0.91786, "ndcg_at_3": 0.95383, "ndcg_at_5": 0.95935, "ndcg_at_10": 0.95935, "ndcg_at_20": 0.96032, "ndcg_at_50": 0.96109, "ndcg_at_100": 0.96109, "map_at_1": 0.91786, "map_at_3": 0.94524, "map_at_5": 0.9481, "map_at_10": 0.9481, "map_at_20": 0.94839, "map_at_50": 0.94854, "map_at_100": 0.94854, "recall_at_1": 0.91786, "recall_at_3": 0.97857, "recall_at_5": 0.99286, "recall_at_10": 0.99286, "recall_at_20": 0.99643, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.91786, "precision_at_3": 0.32619, "precision_at_5": 0.19857, "precision_at_10": 0.09929, "precision_at_20": 0.04982, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.9178571428571428, "mrr_at_3": 0.9452380952380953, "mrr_at_5": 0.948095238095238, "mrr_at_10": 0.948095238095238, "mrr_at_20": 0.9483928571428571, "mrr_at_50": 0.9485416666666666, "mrr_at_100": 0.9485416666666666, "naucs_at_1_max": 0.04487882109365634, "naucs_at_1_std": 0.15499533146591998, "naucs_at_1_diff1": 0.928754110339789, "naucs_at_3_max": 0.8358232181761669, "naucs_at_3_std": 0.9101307189542569, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 0.9346405228758269, "naucs_at_5_std": 0.9346405228758269, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 0.9346405228758269, "naucs_at_10_std": 0.9346405228758269, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": 1.0, "naucs_at_50_std": 1.0, "naucs_at_50_diff1": 1.0, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0 }, "vidore/tatdqa_test": { "ndcg_at_1": 0.70535, "ndcg_at_3": 0.7868, "ndcg_at_5": 0.80621, "ndcg_at_10": 0.82194, "ndcg_at_20": 0.82672, "ndcg_at_50": 0.83053, "ndcg_at_100": 0.83277, "map_at_1": 0.70535, "map_at_3": 0.76742, "map_at_5": 0.77826, "map_at_10": 0.78488, "map_at_20": 0.78628, "map_at_50": 0.78693, "map_at_100": 0.78712, "recall_at_1": 0.70535, "recall_at_3": 0.84265, "recall_at_5": 0.88943, "recall_at_10": 0.93742, "recall_at_20": 0.95565, "recall_at_50": 0.97448, "recall_at_100": 0.98846, "precision_at_1": 0.70535, "precision_at_3": 0.28088, "precision_at_5": 0.17789, "precision_at_10": 0.09374, "precision_at_20": 0.04778, "precision_at_50": 0.01949, "precision_at_100": 0.00988, "mrr_at_1": 0.7035236938031592, "mrr_at_3": 0.767010935601458, "mrr_at_5": 0.7775212636695018, "mrr_at_10": 0.7841125190456905, "mrr_at_20": 0.7855296898659594, "mrr_at_50": 0.7861619428321974, "mrr_at_100": 0.7863629532691659, "naucs_at_1_max": 0.22208704220171518, "naucs_at_1_std": 0.15415493193166072, "naucs_at_1_diff1": 0.85100407554343, "naucs_at_3_max": 0.21518686518713465, "naucs_at_3_std": 0.23043576383644288, "naucs_at_3_diff1": 0.8025394364016292, "naucs_at_5_max": 0.2641753480647899, "naucs_at_5_std": 0.2706978631234192, "naucs_at_5_diff1": 0.7806280461098983, "naucs_at_10_max": 0.2523834187563826, "naucs_at_10_std": 0.32315905277258156, "naucs_at_10_diff1": 0.7532415640618384, "naucs_at_20_max": 0.26510647248798225, "naucs_at_20_std": 0.3525755808522635, "naucs_at_20_diff1": 0.7348455395997588, "naucs_at_50_max": 0.242804665789723, "naucs_at_50_std": 0.44822875988285776, "naucs_at_50_diff1": 0.7286029440909012, "naucs_at_100_max": 0.05932622574605986, "naucs_at_100_std": 0.28624024988935604, "naucs_at_100_diff1": 0.7502131245549767 }, "vidore/shiftproject_test": { "ndcg_at_1": 0.81, "ndcg_at_3": 0.88678, "ndcg_at_5": 0.907, "ndcg_at_10": 0.907, "ndcg_at_20": 0.907, "ndcg_at_50": 0.90904, "ndcg_at_100": 0.90904, "map_at_1": 0.81, "map_at_3": 0.86833, "map_at_5": 0.87933, "map_at_10": 0.87933, "map_at_20": 0.87933, "map_at_50": 0.87968, "map_at_100": 0.87968, "recall_at_1": 0.81, "recall_at_3": 0.94, "recall_at_5": 0.99, "recall_at_10": 0.99, "recall_at_20": 0.99, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.81, "precision_at_3": 0.31333, "precision_at_5": 0.198, "precision_at_10": 0.099, "precision_at_20": 0.0495, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.81, "mrr_at_3": 0.8683333333333334, "mrr_at_5": 0.8793333333333334, "mrr_at_10": 0.8793333333333334, "mrr_at_20": 0.8793333333333334, "mrr_at_50": 0.8796781609195403, "mrr_at_100": 0.8796781609195403, "naucs_at_1_max": -0.18730682592068792, "naucs_at_1_std": -0.7260202210697273, "naucs_at_1_diff1": 0.8433652889098441, "naucs_at_3_max": 0.3544195455960126, "naucs_at_3_std": -0.26914098972922335, "naucs_at_3_diff1": 0.9319172113289744, "naucs_at_5_max": 0.12278244631185926, "naucs_at_5_std": 0.35807656395891135, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 0.12278244631185926, "naucs_at_10_std": 0.35807656395891135, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 0.12278244631185926, "naucs_at_20_std": 0.35807656395891135, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_artificial_intelligence_test": { "ndcg_at_1": 0.99, "ndcg_at_3": 0.99631, "ndcg_at_5": 0.99631, "ndcg_at_10": 0.99631, "ndcg_at_20": 0.99631, "ndcg_at_50": 0.99631, "ndcg_at_100": 0.99631, "map_at_1": 0.99, "map_at_3": 0.995, "map_at_5": 0.995, "map_at_10": 0.995, "map_at_20": 0.995, "map_at_50": 0.995, "map_at_100": 0.995, "recall_at_1": 0.99, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.99, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.99, "mrr_at_3": 0.995, "mrr_at_5": 0.995, "mrr_at_10": 0.995, "mrr_at_20": 0.995, "mrr_at_50": 0.995, "mrr_at_100": 0.995, "naucs_at_1_max": 0.12278244631185359, "naucs_at_1_std": 0.12278244631185359, "naucs_at_1_diff1": 1.0, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_energy_test": { "ndcg_at_1": 0.96, "ndcg_at_3": 0.96631, "ndcg_at_5": 0.96631, "ndcg_at_10": 0.96946, "ndcg_at_20": 0.97209, "ndcg_at_50": 0.97406, "ndcg_at_100": 0.97406, "map_at_1": 0.96, "map_at_3": 0.965, "map_at_5": 0.965, "map_at_10": 0.96625, "map_at_20": 0.96702, "map_at_50": 0.96732, "map_at_100": 0.96732, "recall_at_1": 0.96, "recall_at_3": 0.97, "recall_at_5": 0.97, "recall_at_10": 0.98, "recall_at_20": 0.99, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.96, "precision_at_3": 0.32333, "precision_at_5": 0.194, "precision_at_10": 0.098, "precision_at_20": 0.0495, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.96, "mrr_at_3": 0.965, "mrr_at_5": 0.965, "mrr_at_10": 0.96625, "mrr_at_20": 0.9670192307692308, "mrr_at_50": 0.9673222610722612, "mrr_at_100": 0.9673222610722612, "naucs_at_1_max": 0.7169701213818873, "naucs_at_1_std": -0.03863211951446941, "naucs_at_1_diff1": 1.0, "naucs_at_3_max": 0.7152194211017727, "naucs_at_3_std": -0.34126984126984133, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 0.7152194211017693, "naucs_at_5_std": -0.3412698412698435, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 0.7957516339869297, "naucs_at_10_std": 0.35807656395892185, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 0.7222222222222276, "naucs_at_20_std": 0.35807656395891135, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_government_reports_test": { "ndcg_at_1": 0.95, "ndcg_at_3": 0.97393, "ndcg_at_5": 0.97823, "ndcg_at_10": 0.97823, "ndcg_at_20": 0.97823, "ndcg_at_50": 0.97823, "ndcg_at_100": 0.97823, "map_at_1": 0.95, "map_at_3": 0.96833, "map_at_5": 0.97083, "map_at_10": 0.97083, "map_at_20": 0.97083, "map_at_50": 0.97083, "map_at_100": 0.97083, "recall_at_1": 0.95, "recall_at_3": 0.99, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.95, "precision_at_3": 0.33, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.95, "mrr_at_3": 0.9683333333333333, "mrr_at_5": 0.9708333333333333, "mrr_at_10": 0.9708333333333333, "mrr_at_20": 0.9708333333333333, "mrr_at_50": 0.9708333333333333, "mrr_at_100": 0.9708333333333333, "naucs_at_1_max": 0.6765639589168986, "naucs_at_1_std": 0.5556489262371623, "naucs_at_1_diff1": 0.9738562091503253, "naucs_at_3_max": 1.0, "naucs_at_3_std": 0.8692810457516356, "naucs_at_3_diff1": 0.8692810457516356, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_healthcare_industry_test": { "ndcg_at_1": 0.98, "ndcg_at_3": 0.99262, "ndcg_at_5": 0.99262, "ndcg_at_10": 0.99262, "ndcg_at_20": 0.99262, "ndcg_at_50": 0.99262, "ndcg_at_100": 0.99262, "map_at_1": 0.98, "map_at_3": 0.99, "map_at_5": 0.99, "map_at_10": 0.99, "map_at_20": 0.99, "map_at_50": 0.99, "map_at_100": 0.99, "recall_at_1": 0.98, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.98, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.98, "mrr_at_3": 0.99, "mrr_at_5": 0.99, "mrr_at_10": 0.99, "mrr_at_20": 0.99, "mrr_at_50": 0.99, "mrr_at_100": 0.99, "naucs_at_1_max": 0.6381886087768457, "naucs_at_1_std": -0.14122315592903503, "naucs_at_1_diff1": 1.0, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/synthetic_rse_restaurant_filtered_v1.0_multilingual": { "ndcg_at_1": 0.49561, "ndcg_at_3": 0.53551, "ndcg_at_5": 0.57573, "ndcg_at_10": 0.62797, "ndcg_at_20": 0.66435, "ndcg_at_50": 0.68753, "ndcg_at_100": 0.69778, "map_at_1": 0.24991, "map_at_3": 0.39804, "map_at_5": 0.45596, "map_at_10": 0.50329, "map_at_20": 0.52856, "map_at_50": 0.54302, "map_at_100": 0.54853, "recall_at_1": 0.24991, "recall_at_3": 0.49812, "recall_at_5": 0.62081, "recall_at_10": 0.78505, "recall_at_20": 0.89032, "recall_at_50": 0.94781, "recall_at_100": 0.97423, "precision_at_1": 0.49561, "precision_at_3": 0.37865, "precision_at_5": 0.30965, "precision_at_10": 0.20921, "precision_at_20": 0.13246, "precision_at_50": 0.06912, "precision_at_100": 0.03846, "mrr_at_1": 0.4956140350877193, "mrr_at_3": 0.6271929824561403, "mrr_at_5": 0.642982456140351, "mrr_at_10": 0.6540744221665276, "mrr_at_20": 0.6558978051818611, "mrr_at_50": 0.6561780443192584, "mrr_at_100": 0.6561780443192584, "naucs_at_1_max": -0.07400167664801681, "naucs_at_1_std": 0.03458415022230023, "naucs_at_1_diff1": 0.36837819306504144, "naucs_at_3_max": -0.12444851869211698, "naucs_at_3_std": -0.03350495496687875, "naucs_at_3_diff1": 0.14678568781036586, "naucs_at_5_max": -0.18380991972520577, "naucs_at_5_std": -0.04485546676356389, "naucs_at_5_diff1": 0.09319332805741351, "naucs_at_10_max": -0.2287733254906937, "naucs_at_10_std": -0.11817355407871401, "naucs_at_10_diff1": 0.039989126851164826, "naucs_at_20_max": -0.26978462701811906, "naucs_at_20_std": -0.17072707397422024, "naucs_at_20_diff1": -0.043988113501541394, "naucs_at_50_max": -0.2775075319567234, "naucs_at_50_std": -0.20957246437121108, "naucs_at_50_diff1": -0.1133061107255248, "naucs_at_100_max": -0.27585910810842096, "naucs_at_100_std": -0.2097908968784823, "naucs_at_100_diff1": -0.14037054801741544 }, "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered_multilingual": { "ndcg_at_1": 0.60469, "ndcg_at_3": 0.61057, "ndcg_at_5": 0.63196, "ndcg_at_10": 0.66415, "ndcg_at_20": 0.68919, "ndcg_at_50": 0.71209, "ndcg_at_100": 0.72404, "map_at_1": 0.3749, "map_at_3": 0.50426, "map_at_5": 0.54282, "map_at_10": 0.57771, "map_at_20": 0.59195, "map_at_50": 0.60066, "map_at_100": 0.60317, "recall_at_1": 0.3749, "recall_at_3": 0.56825, "recall_at_5": 0.65837, "recall_at_10": 0.75073, "recall_at_20": 0.81942, "recall_at_50": 0.8876, "recall_at_100": 0.93527, "precision_at_1": 0.60469, "precision_at_3": 0.37448, "precision_at_5": 0.28031, "precision_at_10": 0.17922, "precision_at_20": 0.10523, "precision_at_50": 0.05022, "precision_at_100": 0.02742, "mrr_at_1": 0.6046875, "mrr_at_3": 0.693489583333333, "mrr_at_5": 0.7104427083333327, "mrr_at_10": 0.7158568948412692, "mrr_at_20": 0.7190933507966805, "mrr_at_50": 0.7196874906299467, "mrr_at_100": 0.7198739903840827, "naucs_at_1_max": 0.21052397754348515, "naucs_at_1_std": 0.09375197289505234, "naucs_at_1_diff1": 0.5111101758127156, "naucs_at_3_max": 0.06090465086494804, "naucs_at_3_std": -0.001418024019873419, "naucs_at_3_diff1": -0.03565597745007234, "naucs_at_5_max": 0.004465748594415919, "naucs_at_5_std": -0.0484521722756207, "naucs_at_5_diff1": -0.1245720478106472, "naucs_at_10_max": -0.059499017411910264, "naucs_at_10_std": -0.07086245514678893, "naucs_at_10_diff1": -0.22255807507622197, "naucs_at_20_max": -0.08861071305293747, "naucs_at_20_std": -0.04972647301862899, "naucs_at_20_diff1": -0.28111304038576185, "naucs_at_50_max": -0.0772093711850375, "naucs_at_50_std": -0.03833832084634795, "naucs_at_50_diff1": -0.3229404564565436, "naucs_at_100_max": -0.09326229510606512, "naucs_at_100_std": -0.05851062266000862, "naucs_at_100_diff1": -0.33725273449156307 }, "vidore/synthetics_economics_macro_economy_2024_filtered_v1.0_multilingual": { "ndcg_at_1": 0.65086, "ndcg_at_3": 0.60789, "ndcg_at_5": 0.57982, "ndcg_at_10": 0.56497, "ndcg_at_20": 0.5906, "ndcg_at_50": 0.66142, "ndcg_at_100": 0.6981, "map_at_1": 0.09835, "map_at_3": 0.19265, "map_at_5": 0.2438, "map_at_10": 0.31201, "map_at_20": 0.37105, "map_at_50": 0.43282, "map_at_100": 0.45993, "recall_at_1": 0.09835, "recall_at_3": 0.2383, "recall_at_5": 0.31559, "recall_at_10": 0.44357, "recall_at_20": 0.59021, "recall_at_50": 0.79179, "recall_at_100": 0.90968, "precision_at_1": 0.65086, "precision_at_3": 0.55316, "precision_at_5": 0.49828, "precision_at_10": 0.40603, "precision_at_20": 0.30948, "precision_at_50": 0.1981, "precision_at_100": 0.12724, "mrr_at_1": 0.6508620689655172, "mrr_at_3": 0.7586206896551727, "mrr_at_5": 0.7706896551724141, "mrr_at_10": 0.7741362205801864, "mrr_at_20": 0.7757055120898937, "mrr_at_50": 0.7760672193481526, "mrr_at_100": 0.7760672193481526, "naucs_at_1_max": -0.1577982313304122, "naucs_at_1_std": 0.07083760025943213, "naucs_at_1_diff1": 0.1180440061698451, "naucs_at_3_max": -0.03551684594794198, "naucs_at_3_std": 0.18649544217765762, "naucs_at_3_diff1": 0.006931180468183028, "naucs_at_5_max": -0.042439023438686566, "naucs_at_5_std": 0.1463028288463992, "naucs_at_5_diff1": 0.0052961279206988725, "naucs_at_10_max": -0.014346231321749392, "naucs_at_10_std": 0.13820096240926596, "naucs_at_10_diff1": 0.060959204965535974, "naucs_at_20_max": -0.04036150486209418, "naucs_at_20_std": 0.10161400684234778, "naucs_at_20_diff1": 0.058238772027959955, "naucs_at_50_max": -0.027805254364547293, "naucs_at_50_std": 0.06305093612338106, "naucs_at_50_diff1": 0.014479645829478357, "naucs_at_100_max": -0.04967371405554246, "naucs_at_100_std": -0.0014108802561097272, "naucs_at_100_diff1": 0.004463197803405348 }, "vidore/restaurant_esg_reports_beir": { "ndcg_at_1": 0.66026, "ndcg_at_3": 0.71844, "ndcg_at_5": 0.74746, "ndcg_at_10": 0.78463, "ndcg_at_20": 0.79647, "ndcg_at_50": 0.80898, "ndcg_at_100": 0.81195, "map_at_1": 0.46731, "map_at_3": 0.6321, "map_at_5": 0.67856, "map_at_10": 0.71154, "map_at_20": 0.71984, "map_at_50": 0.72621, "map_at_100": 0.72685, "recall_at_1": 0.46731, "recall_at_3": 0.71218, "recall_at_5": 0.7989, "recall_at_10": 0.89575, "recall_at_20": 0.92767, "recall_at_50": 0.97191, "recall_at_100": 0.981, "precision_at_1": 0.67308, "precision_at_3": 0.41026, "precision_at_5": 0.30769, "precision_at_10": 0.18654, "precision_at_20": 0.10096, "precision_at_50": 0.04462, "precision_at_100": 0.02308, "mrr_at_1": 0.6730769230769231, "mrr_at_3": 0.7756410256410257, "mrr_at_5": 0.7852564102564104, "mrr_at_10": 0.7940705128205129, "mrr_at_20": 0.7940705128205129, "mrr_at_50": 0.7949862637362638, "mrr_at_100": 0.7949862637362638, "naucs_at_1_max": 0.2155210634937776, "naucs_at_1_std": 0.20643949634517547, "naucs_at_1_diff1": 0.5572918449312647, "naucs_at_3_max": -0.22870270992303324, "naucs_at_3_std": -0.16008064610097344, "naucs_at_3_diff1": -0.1052498381156227, "naucs_at_5_max": -0.20249887866652966, "naucs_at_5_std": -0.10577079083523987, "naucs_at_5_diff1": -0.21560021268353965, "naucs_at_10_max": -0.13190257359821772, "naucs_at_10_std": -0.02595056097795122, "naucs_at_10_diff1": -0.25059078573348686, "naucs_at_20_max": -0.10804998986280905, "naucs_at_20_std": 0.009844091874970871, "naucs_at_20_diff1": -0.3106578658424983, "naucs_at_50_max": -0.14217724030501422, "naucs_at_50_std": -0.0010783625765841411, "naucs_at_50_diff1": -0.2900029814490325, "naucs_at_100_max": -0.15203872114868228, "naucs_at_100_std": -0.01643757244869887, "naucs_at_100_diff1": -0.2814362025783248 }, "vidore/synthetic_rse_restaurant_filtered_v1.0": { "ndcg_at_1": 0.52632, "ndcg_at_3": 0.55122, "ndcg_at_5": 0.58777, "ndcg_at_10": 0.63895, "ndcg_at_20": 0.66853, "ndcg_at_50": 0.69944, "ndcg_at_100": 0.7087, "map_at_1": 0.28596, "map_at_3": 0.42149, "map_at_5": 0.48239, "map_at_10": 0.52458, "map_at_20": 0.54701, "map_at_50": 0.56361, "map_at_100": 0.56876, "recall_at_1": 0.28596, "recall_at_3": 0.5136, "recall_at_5": 0.6131, "recall_at_10": 0.77097, "recall_at_20": 0.8575, "recall_at_50": 0.93975, "recall_at_100": 0.96272, "precision_at_1": 0.52632, "precision_at_3": 0.37427, "precision_at_5": 0.31228, "precision_at_10": 0.21053, "precision_at_20": 0.12807, "precision_at_50": 0.06947, "precision_at_100": 0.03842, "mrr_at_1": 0.5263157894736842, "mrr_at_3": 0.6461988304093568, "mrr_at_5": 0.6549707602339182, "mrr_at_10": 0.6659844054580898, "mrr_at_20": 0.6673339331234068, "mrr_at_50": 0.6679388938009629, "mrr_at_100": 0.6679388938009629, "naucs_at_1_max": -0.018448353550608226, "naucs_at_1_std": 0.0821493132969922, "naucs_at_1_diff1": 0.30290509894323814, "naucs_at_3_max": -0.16441638665467292, "naucs_at_3_std": 0.05181451792946125, "naucs_at_3_diff1": 0.17081667434507056, "naucs_at_5_max": -0.2223104695809391, "naucs_at_5_std": -0.0034901998225501346, "naucs_at_5_diff1": 0.11827209266301716, "naucs_at_10_max": -0.3490810408001137, "naucs_at_10_std": -0.10137127012539782, "naucs_at_10_diff1": 0.024290790916341346, "naucs_at_20_max": -0.41179472120133376, "naucs_at_20_std": -0.18184189514711724, "naucs_at_20_diff1": -0.026948094739752244, "naucs_at_50_max": -0.4169831140986695, "naucs_at_50_std": -0.2419827968566681, "naucs_at_50_diff1": -0.14707184113572777, "naucs_at_100_max": -0.41046730276590754, "naucs_at_100_std": -0.2436064377498405, "naucs_at_100_diff1": -0.16240322922206768 }, "vidore/synthetic_economics_macro_economy_2024_filtered_v1.0": { "ndcg_at_1": 0.7931, "ndcg_at_3": 0.69066, "ndcg_at_5": 0.65993, "ndcg_at_10": 0.62291, "ndcg_at_20": 0.64258, "ndcg_at_50": 0.70498, "ndcg_at_100": 0.73984, "map_at_1": 0.11768, "map_at_3": 0.22479, "map_at_5": 0.28825, "map_at_10": 0.35934, "map_at_20": 0.42423, "map_at_50": 0.48412, "map_at_100": 0.51142, "recall_at_1": 0.11768, "recall_at_3": 0.26124, "recall_at_5": 0.34909, "recall_at_10": 0.46824, "recall_at_20": 0.62417, "recall_at_50": 0.80904, "recall_at_100": 0.92098, "precision_at_1": 0.7931, "precision_at_3": 0.62069, "precision_at_5": 0.56552, "precision_at_10": 0.43966, "precision_at_20": 0.325, "precision_at_50": 0.20172, "precision_at_100": 0.12914, "mrr_at_1": 0.7931034482758621, "mrr_at_3": 0.8534482758620691, "mrr_at_5": 0.8568965517241378, "mrr_at_10": 0.8597701149425288, "mrr_at_20": 0.8620158408190863, "mrr_at_50": 0.8620158408190863, "mrr_at_100": 0.8620158408190863, "naucs_at_1_max": 0.271935938073248, "naucs_at_1_std": 0.43674674991288676, "naucs_at_1_diff1": 0.3617820074090711, "naucs_at_3_max": 0.045282507585294354, "naucs_at_3_std": 0.14844039042832088, "naucs_at_3_diff1": 0.11059599624516278, "naucs_at_5_max": 0.06086984174322351, "naucs_at_5_std": 0.15073825039690825, "naucs_at_5_diff1": 0.07597281303252662, "naucs_at_10_max": 0.030803253605626704, "naucs_at_10_std": 0.12023189876899391, "naucs_at_10_diff1": 0.09409994930103974, "naucs_at_20_max": 0.06111477495291343, "naucs_at_20_std": 0.1377686346379398, "naucs_at_20_diff1": 0.08342034215323584, "naucs_at_50_max": -0.0214422324455313, "naucs_at_50_std": 0.0629645228435432, "naucs_at_50_diff1": 0.03433912496739525, "naucs_at_100_max": -0.07270216260742496, "naucs_at_100_std": -0.009713954266926159, "naucs_at_100_diff1": -0.00289458364944889 }, "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered": { "ndcg_at_1": 0.63125, "ndcg_at_3": 0.63272, "ndcg_at_5": 0.65663, "ndcg_at_10": 0.68997, "ndcg_at_20": 0.71467, "ndcg_at_50": 0.73633, "ndcg_at_100": 0.74718, "map_at_1": 0.39991, "map_at_3": 0.5245, "map_at_5": 0.57059, "map_at_10": 0.6074, "map_at_20": 0.62094, "map_at_50": 0.62966, "map_at_100": 0.63202, "recall_at_1": 0.39991, "recall_at_3": 0.57796, "recall_at_5": 0.67437, "recall_at_10": 0.77096, "recall_at_20": 0.84121, "recall_at_50": 0.90687, "recall_at_100": 0.94874, "precision_at_1": 0.63125, "precision_at_3": 0.38958, "precision_at_5": 0.29375, "precision_at_10": 0.18688, "precision_at_20": 0.10844, "precision_at_50": 0.05075, "precision_at_100": 0.02756, "mrr_at_1": 0.63125, "mrr_at_3": 0.715625, "mrr_at_5": 0.7274999999999998, "mrr_at_10": 0.7328447420634919, "mrr_at_20": 0.7370251189782439, "mrr_at_50": 0.7376058481449105, "mrr_at_100": 0.737786021875342, "naucs_at_1_max": 0.40116615786322174, "naucs_at_1_std": 0.12804177750728857, "naucs_at_1_diff1": 0.5318484780069298, "naucs_at_3_max": 0.031070174638639564, "naucs_at_3_std": -0.03258401349687653, "naucs_at_3_diff1": -0.08211207879257611, "naucs_at_5_max": -0.03804589294380602, "naucs_at_5_std": -0.0638529839983047, "naucs_at_5_diff1": -0.1625118765251509, "naucs_at_10_max": -0.08589738686452425, "naucs_at_10_std": -0.0652662376777225, "naucs_at_10_diff1": -0.27588514824978205, "naucs_at_20_max": -0.14280800843691288, "naucs_at_20_std": -0.050772727555263505, "naucs_at_20_diff1": -0.33260125020460346, "naucs_at_50_max": -0.18794085961226306, "naucs_at_50_std": -0.09315309781822757, "naucs_at_50_diff1": -0.3809855756340824, "naucs_at_100_max": -0.20443798069201818, "naucs_at_100_std": -0.09564469982214785, "naucs_at_100_diff1": -0.416619080759233 } } }