diff --git a/results/BAAI__bge-m3/5617a9f61b028005a4858fdac845db406aefb181/ElasticKBRetrieval.json b/results/BAAI__bge-m3/5617a9f61b028005a4858fdac845db406aefb181/ElasticKBRetrieval.json new file mode 100644 index 0000000000..5148ad304f --- /dev/null +++ b/results/BAAI__bge-m3/5617a9f61b028005a4858fdac845db406aefb181/ElasticKBRetrieval.json @@ -0,0 +1,324 @@ +{ + "dataset_revision": "21bdf1e024bf7c9f46720017559ce2f8c6116507", + "task_name": "ElasticKBRetrieval", + "mteb_version": "2.12.38", + "scores": { + "synthetic_test": [ + { + "ndcg_at_1": 0.70213, + "ndcg_at_3": 0.56287, + "ndcg_at_5": 0.53684, + "ndcg_at_10": 0.51354, + "ndcg_at_20": 0.51233, + "ndcg_at_100": 0.55395, + "ndcg_at_1000": 0.61713, + "map_at_1": 0.22897, + "map_at_3": 0.28765, + "map_at_5": 0.3055, + "map_at_10": 0.32613, + "map_at_20": 0.34619, + "map_at_100": 0.37307, + "map_at_1000": 0.38407, + "recall_at_1": 0.22897, + "recall_at_3": 0.3188, + "recall_at_5": 0.35703, + "recall_at_10": 0.40616, + "recall_at_20": 0.47311, + "recall_at_100": 0.64121, + "recall_at_1000": 0.84787, + "accuracy": 0.22897, + "precision_at_1": 0.70213, + "precision_at_3": 0.41489, + "precision_at_5": 0.32979, + "precision_at_10": 0.23245, + "precision_at_20": 0.16755, + "precision_at_100": 0.06553, + "precision_at_1000": 0.01234, + "mrr_at_1": 0.702128, + "mrr_at_3": 0.782801, + "mrr_at_5": 0.793706, + "mrr_at_10": 0.797716, + "mrr_at_20": 0.799809, + "mrr_at_100": 0.800236, + "mrr_at_1000": 0.800236, + "nauc_ndcg_at_1_max": 0.03637, + "nauc_ndcg_at_1_std": -0.155315, + "nauc_ndcg_at_1_diff1": 0.393593, + "nauc_ndcg_at_3_max": 0.101207, + "nauc_ndcg_at_3_std": 0.059933, + "nauc_ndcg_at_3_diff1": 0.133793, + "nauc_ndcg_at_5_max": 0.10129, + "nauc_ndcg_at_5_std": 0.063464, + "nauc_ndcg_at_5_diff1": 0.172103, + "nauc_ndcg_at_10_max": 0.13233, + "nauc_ndcg_at_10_std": 0.08067, + "nauc_ndcg_at_10_diff1": 0.262056, + "nauc_ndcg_at_20_max": 0.177488, + "nauc_ndcg_at_20_std": 0.101258, + "nauc_ndcg_at_20_diff1": 0.292539, + "nauc_ndcg_at_100_max": 0.238721, + "nauc_ndcg_at_100_std": 0.174771, + "nauc_ndcg_at_100_diff1": 0.304403, + "nauc_ndcg_at_1000_max": 0.223684, + "nauc_ndcg_at_1000_std": 0.186361, + "nauc_ndcg_at_1000_diff1": 0.266568, + "nauc_map_at_1_max": 0.09017, + "nauc_map_at_1_std": -0.011468, + "nauc_map_at_1_diff1": 0.49284, + "nauc_map_at_3_max": 0.122122, + "nauc_map_at_3_std": -0.012008, + "nauc_map_at_3_diff1": 0.428711, + "nauc_map_at_5_max": 0.119363, + "nauc_map_at_5_std": -0.013921, + "nauc_map_at_5_diff1": 0.41813, + "nauc_map_at_10_max": 0.139708, + "nauc_map_at_10_std": 0.006488, + "nauc_map_at_10_diff1": 0.406346, + "nauc_map_at_20_max": 0.178808, + "nauc_map_at_20_std": 0.052469, + "nauc_map_at_20_diff1": 0.374139, + "nauc_map_at_100_max": 0.20081, + "nauc_map_at_100_std": 0.100564, + "nauc_map_at_100_diff1": 0.348661, + "nauc_map_at_1000_max": 0.197797, + "nauc_map_at_1000_std": 0.109939, + "nauc_map_at_1000_diff1": 0.340116, + "nauc_recall_at_1_max": 0.09017, + "nauc_recall_at_1_std": -0.011468, + "nauc_recall_at_1_diff1": 0.49284, + "nauc_recall_at_3_max": 0.106298, + "nauc_recall_at_3_std": -0.002766, + "nauc_recall_at_3_diff1": 0.366062, + "nauc_recall_at_5_max": 0.097266, + "nauc_recall_at_5_std": -0.015098, + "nauc_recall_at_5_diff1": 0.357176, + "nauc_recall_at_10_max": 0.13274, + "nauc_recall_at_10_std": 0.018272, + "nauc_recall_at_10_diff1": 0.350881, + "nauc_recall_at_20_max": 0.198021, + "nauc_recall_at_20_std": 0.098643, + "nauc_recall_at_20_diff1": 0.266127, + "nauc_recall_at_100_max": 0.237467, + "nauc_recall_at_100_std": 0.235715, + "nauc_recall_at_100_diff1": 0.219191, + "nauc_recall_at_1000_max": 0.194755, + "nauc_recall_at_1000_std": 0.259497, + "nauc_recall_at_1000_diff1": 0.214908, + "nauc_precision_at_1_max": 0.03637, + "nauc_precision_at_1_std": -0.155315, + "nauc_precision_at_1_diff1": 0.393593, + "nauc_precision_at_3_max": 0.102194, + "nauc_precision_at_3_std": 0.115327, + "nauc_precision_at_3_diff1": -0.232675, + "nauc_precision_at_5_max": 0.051921, + "nauc_precision_at_5_std": 0.125716, + "nauc_precision_at_5_diff1": -0.311005, + "nauc_precision_at_10_max": 0.078458, + "nauc_precision_at_10_std": 0.209922, + "nauc_precision_at_10_diff1": -0.328814, + "nauc_precision_at_20_max": 0.134614, + "nauc_precision_at_20_std": 0.300578, + "nauc_precision_at_20_diff1": -0.390179, + "nauc_precision_at_100_max": -0.012795, + "nauc_precision_at_100_std": 0.249381, + "nauc_precision_at_100_diff1": -0.371273, + "nauc_precision_at_1000_max": -0.173382, + "nauc_precision_at_1000_std": 0.063768, + "nauc_precision_at_1000_diff1": -0.303265, + "nauc_mrr_at_1_max": 0.03637, + "nauc_mrr_at_1_std": -0.155315, + "nauc_mrr_at_1_diff1": 0.393593, + "nauc_mrr_at_3_max": 0.087938, + "nauc_mrr_at_3_std": -0.079861, + "nauc_mrr_at_3_diff1": 0.326316, + "nauc_mrr_at_5_max": 0.080416, + "nauc_mrr_at_5_std": -0.089234, + "nauc_mrr_at_5_diff1": 0.347435, + "nauc_mrr_at_10_max": 0.084397, + "nauc_mrr_at_10_std": -0.089832, + "nauc_mrr_at_10_diff1": 0.359597, + "nauc_mrr_at_20_max": 0.079692, + "nauc_mrr_at_20_std": -0.094003, + "nauc_mrr_at_20_diff1": 0.354783, + "nauc_mrr_at_100_max": 0.078293, + "nauc_mrr_at_100_std": -0.094719, + "nauc_mrr_at_100_diff1": 0.354029, + "nauc_mrr_at_1000_max": 0.078293, + "nauc_mrr_at_1000_std": -0.094719, + "nauc_mrr_at_1000_diff1": 0.354029, + "hit_rate_at_1": 0.70213, + "hit_rate_at_3": 0.87766, + "hit_rate_at_5": 0.92553, + "hit_rate_at_10": 0.95213, + "hit_rate_at_20": 0.98404, + "hit_rate_at_100": 1.0, + "hit_rate_at_1000": 1.0, + "main_score": 0.51354, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ], + "real_chat_test": [ + { + "ndcg_at_1": 0.73276, + "ndcg_at_3": 0.65531, + "ndcg_at_5": 0.61008, + "ndcg_at_10": 0.55386, + "ndcg_at_20": 0.51278, + "ndcg_at_100": 0.46226, + "ndcg_at_1000": 0.54149, + "map_at_1": 0.05417, + "map_at_3": 0.09937, + "map_at_5": 0.11647, + "map_at_10": 0.13877, + "map_at_20": 0.15862, + "map_at_100": 0.2014, + "map_at_1000": 0.25157, + "recall_at_1": 0.05417, + "recall_at_3": 0.11056, + "recall_at_5": 0.13696, + "recall_at_10": 0.17956, + "recall_at_20": 0.22769, + "recall_at_100": 0.37549, + "recall_at_1000": 0.69322, + "accuracy": 0.05417, + "precision_at_1": 0.73276, + "precision_at_3": 0.62069, + "precision_at_5": 0.54828, + "precision_at_10": 0.45216, + "precision_at_20": 0.36789, + "precision_at_100": 0.21707, + "precision_at_1000": 0.07642, + "mrr_at_1": 0.732759, + "mrr_at_3": 0.801724, + "mrr_at_5": 0.809483, + "mrr_at_10": 0.814426, + "mrr_at_20": 0.816045, + "mrr_at_100": 0.816574, + "mrr_at_1000": 0.816599, + "nauc_ndcg_at_1_max": 0.392414, + "nauc_ndcg_at_1_std": 0.239732, + "nauc_ndcg_at_1_diff1": 0.408702, + "nauc_ndcg_at_3_max": 0.31895, + "nauc_ndcg_at_3_std": 0.168499, + "nauc_ndcg_at_3_diff1": 0.194345, + "nauc_ndcg_at_5_max": 0.311744, + "nauc_ndcg_at_5_std": 0.183467, + "nauc_ndcg_at_5_diff1": 0.140404, + "nauc_ndcg_at_10_max": 0.338781, + "nauc_ndcg_at_10_std": 0.251598, + "nauc_ndcg_at_10_diff1": 0.137225, + "nauc_ndcg_at_20_max": 0.350291, + "nauc_ndcg_at_20_std": 0.276095, + "nauc_ndcg_at_20_diff1": 0.148665, + "nauc_ndcg_at_100_max": 0.433396, + "nauc_ndcg_at_100_std": 0.28703, + "nauc_ndcg_at_100_diff1": 0.243558, + "nauc_ndcg_at_1000_max": 0.423155, + "nauc_ndcg_at_1000_std": 0.328531, + "nauc_ndcg_at_1000_diff1": 0.255891, + "nauc_map_at_1_max": 0.372885, + "nauc_map_at_1_std": -0.034218, + "nauc_map_at_1_diff1": 0.564564, + "nauc_map_at_3_max": 0.323696, + "nauc_map_at_3_std": -0.087526, + "nauc_map_at_3_diff1": 0.428836, + "nauc_map_at_5_max": 0.323596, + "nauc_map_at_5_std": -0.095468, + "nauc_map_at_5_diff1": 0.414677, + "nauc_map_at_10_max": 0.351467, + "nauc_map_at_10_std": -0.051137, + "nauc_map_at_10_diff1": 0.405714, + "nauc_map_at_20_max": 0.358121, + "nauc_map_at_20_std": -0.024856, + "nauc_map_at_20_diff1": 0.390702, + "nauc_map_at_100_max": 0.405742, + "nauc_map_at_100_std": 0.093437, + "nauc_map_at_100_diff1": 0.350073, + "nauc_map_at_1000_max": 0.4367, + "nauc_map_at_1000_std": 0.226577, + "nauc_map_at_1000_diff1": 0.299833, + "nauc_recall_at_1_max": 0.372885, + "nauc_recall_at_1_std": -0.034218, + "nauc_recall_at_1_diff1": 0.564564, + "nauc_recall_at_3_max": 0.26701, + "nauc_recall_at_3_std": -0.114567, + "nauc_recall_at_3_diff1": 0.380972, + "nauc_recall_at_5_max": 0.251532, + "nauc_recall_at_5_std": -0.126484, + "nauc_recall_at_5_diff1": 0.356845, + "nauc_recall_at_10_max": 0.297609, + "nauc_recall_at_10_std": -0.049554, + "nauc_recall_at_10_diff1": 0.349745, + "nauc_recall_at_20_max": 0.289278, + "nauc_recall_at_20_std": -0.024984, + "nauc_recall_at_20_diff1": 0.316856, + "nauc_recall_at_100_max": 0.310333, + "nauc_recall_at_100_std": 0.110186, + "nauc_recall_at_100_diff1": 0.245096, + "nauc_recall_at_1000_max": 0.286375, + "nauc_recall_at_1000_std": 0.283108, + "nauc_recall_at_1000_diff1": 0.152341, + "nauc_precision_at_1_max": 0.392414, + "nauc_precision_at_1_std": 0.239732, + "nauc_precision_at_1_diff1": 0.408702, + "nauc_precision_at_3_max": 0.263014, + "nauc_precision_at_3_std": 0.165473, + "nauc_precision_at_3_diff1": 0.082503, + "nauc_precision_at_5_max": 0.228145, + "nauc_precision_at_5_std": 0.192415, + "nauc_precision_at_5_diff1": -0.016206, + "nauc_precision_at_10_max": 0.19358, + "nauc_precision_at_10_std": 0.281694, + "nauc_precision_at_10_diff1": -0.094217, + "nauc_precision_at_20_max": 0.11416, + "nauc_precision_at_20_std": 0.329075, + "nauc_precision_at_20_diff1": -0.169462, + "nauc_precision_at_100_max": 0.079117, + "nauc_precision_at_100_std": 0.372067, + "nauc_precision_at_100_diff1": -0.195606, + "nauc_precision_at_1000_max": 0.005204, + "nauc_precision_at_1000_std": 0.233734, + "nauc_precision_at_1000_diff1": -0.169497, + "nauc_mrr_at_1_max": 0.392414, + "nauc_mrr_at_1_std": 0.239732, + "nauc_mrr_at_1_diff1": 0.408702, + "nauc_mrr_at_3_max": 0.41073, + "nauc_mrr_at_3_std": 0.232272, + "nauc_mrr_at_3_diff1": 0.409098, + "nauc_mrr_at_5_max": 0.397771, + "nauc_mrr_at_5_std": 0.225726, + "nauc_mrr_at_5_diff1": 0.4051, + "nauc_mrr_at_10_max": 0.407635, + "nauc_mrr_at_10_std": 0.224537, + "nauc_mrr_at_10_diff1": 0.410387, + "nauc_mrr_at_20_max": 0.407774, + "nauc_mrr_at_20_std": 0.224568, + "nauc_mrr_at_20_diff1": 0.410494, + "nauc_mrr_at_100_max": 0.407187, + "nauc_mrr_at_100_std": 0.226996, + "nauc_mrr_at_100_diff1": 0.409546, + "nauc_mrr_at_1000_max": 0.407195, + "nauc_mrr_at_1000_std": 0.226923, + "nauc_mrr_at_1000_diff1": 0.409689, + "hit_rate_at_1": 0.73276, + "hit_rate_at_3": 0.88793, + "hit_rate_at_5": 0.92241, + "hit_rate_at_10": 0.9569, + "hit_rate_at_20": 0.97845, + "hit_rate_at_100": 0.99569, + "hit_rate_at_1000": 1.0, + "main_score": 0.55386, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 470.54601192474365, + "kg_co2_emissions": null, + "date": 1777843190.525151 +} \ No newline at end of file diff --git a/results/BAAI__bge-m3/5617a9f61b028005a4858fdac845db406aefb181/GlobalNewsRetrieval.json b/results/BAAI__bge-m3/5617a9f61b028005a4858fdac845db406aefb181/GlobalNewsRetrieval.json new file mode 100644 index 0000000000..469d810fe0 --- /dev/null +++ b/results/BAAI__bge-m3/5617a9f61b028005a4858fdac845db406aefb181/GlobalNewsRetrieval.json @@ -0,0 +1,1562 @@ +{ + "dataset_revision": "234dcd32922d32c78e08bfe506a2b90a5401cf7c", + "task_name": "GlobalNewsRetrieval", + "mteb_version": "2.12.38", + "scores": { + "test": [ + { + "ndcg_at_1": 0.66541, + "ndcg_at_3": 0.73829, + "ndcg_at_5": 0.76071, + "ndcg_at_10": 0.77694, + "ndcg_at_20": 0.78839, + "ndcg_at_100": 0.80017, + "ndcg_at_1000": 0.80183, + "map_at_1": 0.63179, + "map_at_3": 0.71147, + "map_at_5": 0.72578, + "map_at_10": 0.73363, + "map_at_20": 0.73742, + "map_at_100": 0.73945, + "map_at_1000": 0.73954, + "recall_at_1": 0.63179, + "recall_at_3": 0.79177, + "recall_at_5": 0.84294, + "recall_at_10": 0.88993, + "recall_at_20": 0.93108, + "recall_at_100": 0.98872, + "recall_at_1000": 1.0, + "accuracy": 0.63179, + "precision_at_1": 0.66541, + "precision_at_3": 0.28613, + "precision_at_5": 0.18546, + "precision_at_10": 0.09887, + "precision_at_20": 0.05213, + "precision_at_100": 0.01117, + "precision_at_1000": 0.00113, + "mrr_at_1": 0.665414, + "mrr_at_3": 0.736007, + "mrr_at_5": 0.746094, + "mrr_at_10": 0.751357, + "mrr_at_20": 0.754118, + "mrr_at_100": 0.755604, + "mrr_at_1000": 0.755667, + "nauc_ndcg_at_1_max": 0.381906, + "nauc_ndcg_at_1_std": -0.258191, + "nauc_ndcg_at_1_diff1": 0.74922, + "nauc_ndcg_at_3_max": 0.394746, + "nauc_ndcg_at_3_std": -0.295354, + "nauc_ndcg_at_3_diff1": 0.725501, + "nauc_ndcg_at_5_max": 0.40397, + "nauc_ndcg_at_5_std": -0.303726, + "nauc_ndcg_at_5_diff1": 0.729551, + "nauc_ndcg_at_10_max": 0.403322, + "nauc_ndcg_at_10_std": -0.307298, + "nauc_ndcg_at_10_diff1": 0.732626, + "nauc_ndcg_at_20_max": 0.417632, + "nauc_ndcg_at_20_std": -0.283654, + "nauc_ndcg_at_20_diff1": 0.735774, + "nauc_ndcg_at_100_max": 0.409767, + "nauc_ndcg_at_100_std": -0.284404, + "nauc_ndcg_at_100_diff1": 0.735841, + "nauc_ndcg_at_1000_max": 0.405683, + "nauc_ndcg_at_1000_std": -0.289175, + "nauc_ndcg_at_1000_diff1": 0.73516, + "nauc_map_at_1_max": 0.330744, + "nauc_map_at_1_std": -0.257663, + "nauc_map_at_1_diff1": 0.733207, + "nauc_map_at_3_max": 0.376897, + "nauc_map_at_3_std": -0.286639, + "nauc_map_at_3_diff1": 0.728086, + "nauc_map_at_5_max": 0.387528, + "nauc_map_at_5_std": -0.291615, + "nauc_map_at_5_diff1": 0.731705, + "nauc_map_at_10_max": 0.388534, + "nauc_map_at_10_std": -0.29408, + "nauc_map_at_10_diff1": 0.732839, + "nauc_map_at_20_max": 0.391893, + "nauc_map_at_20_std": -0.287861, + "nauc_map_at_20_diff1": 0.733388, + "nauc_map_at_100_max": 0.390817, + "nauc_map_at_100_std": -0.288495, + "nauc_map_at_100_diff1": 0.733426, + "nauc_map_at_1000_max": 0.390638, + "nauc_map_at_1000_std": -0.288707, + "nauc_map_at_1000_diff1": 0.733397, + "nauc_recall_at_1_max": 0.330744, + "nauc_recall_at_1_std": -0.257663, + "nauc_recall_at_1_diff1": 0.733207, + "nauc_recall_at_3_max": 0.406219, + "nauc_recall_at_3_std": -0.32563, + "nauc_recall_at_3_diff1": 0.694774, + "nauc_recall_at_5_max": 0.429722, + "nauc_recall_at_5_std": -0.354195, + "nauc_recall_at_5_diff1": 0.695212, + "nauc_recall_at_10_max": 0.433239, + "nauc_recall_at_10_std": -0.384828, + "nauc_recall_at_10_diff1": 0.709327, + "nauc_recall_at_20_max": 0.589358, + "nauc_recall_at_20_std": -0.190781, + "nauc_recall_at_20_diff1": 0.731304, + "nauc_recall_at_100_max": 0.829356, + "nauc_recall_at_100_std": 0.225234, + "nauc_recall_at_100_diff1": 0.785019, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.381906, + "nauc_precision_at_1_std": -0.258191, + "nauc_precision_at_1_diff1": 0.74922, + "nauc_precision_at_3_max": 0.405527, + "nauc_precision_at_3_std": -0.234794, + "nauc_precision_at_3_diff1": 0.468903, + "nauc_precision_at_5_max": 0.346595, + "nauc_precision_at_5_std": -0.196902, + "nauc_precision_at_5_diff1": 0.309271, + "nauc_precision_at_10_max": 0.273355, + "nauc_precision_at_10_std": -0.14583, + "nauc_precision_at_10_diff1": 0.159709, + "nauc_precision_at_20_max": 0.244657, + "nauc_precision_at_20_std": 0.001985, + "nauc_precision_at_20_diff1": 0.011611, + "nauc_precision_at_100_max": 0.086321, + "nauc_precision_at_100_std": 0.099012, + "nauc_precision_at_100_diff1": -0.223448, + "nauc_precision_at_1000_max": 0.038317, + "nauc_precision_at_1000_std": 0.083936, + "nauc_precision_at_1000_diff1": -0.267931, + "nauc_mrr_at_1_max": 0.381906, + "nauc_mrr_at_1_std": -0.258191, + "nauc_mrr_at_1_diff1": 0.74922, + "nauc_mrr_at_3_max": 0.413794, + "nauc_mrr_at_3_std": -0.283093, + "nauc_mrr_at_3_diff1": 0.73751, + "nauc_mrr_at_5_max": 0.410117, + "nauc_mrr_at_5_std": -0.285208, + "nauc_mrr_at_5_diff1": 0.737438, + "nauc_mrr_at_10_max": 0.407323, + "nauc_mrr_at_10_std": -0.286664, + "nauc_mrr_at_10_diff1": 0.738681, + "nauc_mrr_at_20_max": 0.410489, + "nauc_mrr_at_20_std": -0.282248, + "nauc_mrr_at_20_diff1": 0.739855, + "nauc_mrr_at_100_max": 0.409991, + "nauc_mrr_at_100_std": -0.282217, + "nauc_mrr_at_100_diff1": 0.740109, + "nauc_mrr_at_1000_max": 0.409837, + "nauc_mrr_at_1000_std": -0.282387, + "nauc_mrr_at_1000_diff1": 0.740061, + "hit_rate_at_1": 0.66541, + "hit_rate_at_3": 0.8208, + "hit_rate_at_5": 0.86591, + "hit_rate_at_10": 0.90602, + "hit_rate_at_20": 0.94236, + "hit_rate_at_100": 0.99123, + "hit_rate_at_1000": 1.0, + "main_score": 0.77694, + "hf_subset": "ar", + "languages": [ + "ara-Arab" + ] + }, + { + "ndcg_at_1": 0.77634, + "ndcg_at_3": 0.82432, + "ndcg_at_5": 0.84035, + "ndcg_at_10": 0.85167, + "ndcg_at_20": 0.85806, + "ndcg_at_100": 0.86729, + "ndcg_at_1000": 0.86798, + "map_at_1": 0.71819, + "map_at_3": 0.80111, + "map_at_5": 0.81186, + "map_at_10": 0.8181, + "map_at_20": 0.82023, + "map_at_100": 0.82188, + "map_at_1000": 0.82194, + "recall_at_1": 0.71819, + "recall_at_3": 0.86263, + "recall_at_5": 0.89794, + "recall_at_10": 0.92921, + "recall_at_20": 0.95188, + "recall_at_100": 0.99606, + "recall_at_1000": 1.0, + "accuracy": 0.71819, + "precision_at_1": 0.77634, + "precision_at_3": 0.32545, + "precision_at_5": 0.20559, + "precision_at_10": 0.10742, + "precision_at_20": 0.05543, + "precision_at_100": 0.01175, + "precision_at_1000": 0.00118, + "mrr_at_1": 0.776344, + "mrr_at_3": 0.831004, + "mrr_at_5": 0.837616, + "mrr_at_10": 0.840276, + "mrr_at_20": 0.84162, + "mrr_at_100": 0.842561, + "mrr_at_1000": 0.84257, + "nauc_ndcg_at_1_max": 0.482033, + "nauc_ndcg_at_1_std": -0.449457, + "nauc_ndcg_at_1_diff1": 0.819855, + "nauc_ndcg_at_3_max": 0.449339, + "nauc_ndcg_at_3_std": -0.566727, + "nauc_ndcg_at_3_diff1": 0.776722, + "nauc_ndcg_at_5_max": 0.447515, + "nauc_ndcg_at_5_std": -0.569869, + "nauc_ndcg_at_5_diff1": 0.776738, + "nauc_ndcg_at_10_max": 0.470007, + "nauc_ndcg_at_10_std": -0.582066, + "nauc_ndcg_at_10_diff1": 0.784737, + "nauc_ndcg_at_20_max": 0.468429, + "nauc_ndcg_at_20_std": -0.575166, + "nauc_ndcg_at_20_diff1": 0.788939, + "nauc_ndcg_at_100_max": 0.467188, + "nauc_ndcg_at_100_std": -0.550351, + "nauc_ndcg_at_100_diff1": 0.789379, + "nauc_ndcg_at_1000_max": 0.465691, + "nauc_ndcg_at_1000_std": -0.547283, + "nauc_ndcg_at_1000_diff1": 0.788362, + "nauc_map_at_1_max": 0.386142, + "nauc_map_at_1_std": -0.430678, + "nauc_map_at_1_diff1": 0.792326, + "nauc_map_at_3_max": 0.441094, + "nauc_map_at_3_std": -0.534931, + "nauc_map_at_3_diff1": 0.784486, + "nauc_map_at_5_max": 0.445096, + "nauc_map_at_5_std": -0.537245, + "nauc_map_at_5_diff1": 0.785224, + "nauc_map_at_10_max": 0.457818, + "nauc_map_at_10_std": -0.542393, + "nauc_map_at_10_diff1": 0.788697, + "nauc_map_at_20_max": 0.456551, + "nauc_map_at_20_std": -0.541604, + "nauc_map_at_20_diff1": 0.789645, + "nauc_map_at_100_max": 0.456382, + "nauc_map_at_100_std": -0.536974, + "nauc_map_at_100_diff1": 0.789493, + "nauc_map_at_1000_max": 0.4563, + "nauc_map_at_1000_std": -0.53678, + "nauc_map_at_1000_diff1": 0.789432, + "nauc_recall_at_1_max": 0.386142, + "nauc_recall_at_1_std": -0.430678, + "nauc_recall_at_1_diff1": 0.792326, + "nauc_recall_at_3_max": 0.420655, + "nauc_recall_at_3_std": -0.667792, + "nauc_recall_at_3_diff1": 0.724541, + "nauc_recall_at_5_max": 0.394253, + "nauc_recall_at_5_std": -0.700644, + "nauc_recall_at_5_diff1": 0.703344, + "nauc_recall_at_10_max": 0.481851, + "nauc_recall_at_10_std": -0.820649, + "nauc_recall_at_10_diff1": 0.724408, + "nauc_recall_at_20_max": 0.493433, + "nauc_recall_at_20_std": -0.869138, + "nauc_recall_at_20_diff1": 0.762293, + "nauc_recall_at_100_max": 0.713819, + "nauc_recall_at_100_std": -1.036563, + "nauc_recall_at_100_diff1": 0.903446, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.482033, + "nauc_precision_at_1_std": -0.449457, + "nauc_precision_at_1_diff1": 0.819855, + "nauc_precision_at_3_max": 0.279348, + "nauc_precision_at_3_std": -0.284804, + "nauc_precision_at_3_diff1": 0.167979, + "nauc_precision_at_5_max": 0.195017, + "nauc_precision_at_5_std": -0.174902, + "nauc_precision_at_5_diff1": 0.020868, + "nauc_precision_at_10_max": 0.166761, + "nauc_precision_at_10_std": -0.094552, + "nauc_precision_at_10_diff1": -0.082794, + "nauc_precision_at_20_max": 0.107307, + "nauc_precision_at_20_std": -0.003874, + "nauc_precision_at_20_diff1": -0.165342, + "nauc_precision_at_100_max": -0.010283, + "nauc_precision_at_100_std": 0.211369, + "nauc_precision_at_100_diff1": -0.34304, + "nauc_precision_at_1000_max": -0.028308, + "nauc_precision_at_1000_std": 0.237074, + "nauc_precision_at_1000_diff1": -0.357685, + "nauc_mrr_at_1_max": 0.482033, + "nauc_mrr_at_1_std": -0.449457, + "nauc_mrr_at_1_diff1": 0.819855, + "nauc_mrr_at_3_max": 0.481772, + "nauc_mrr_at_3_std": -0.539456, + "nauc_mrr_at_3_diff1": 0.79142, + "nauc_mrr_at_5_max": 0.474327, + "nauc_mrr_at_5_std": -0.534884, + "nauc_mrr_at_5_diff1": 0.792013, + "nauc_mrr_at_10_max": 0.477412, + "nauc_mrr_at_10_std": -0.537483, + "nauc_mrr_at_10_diff1": 0.794381, + "nauc_mrr_at_20_max": 0.477941, + "nauc_mrr_at_20_std": -0.534703, + "nauc_mrr_at_20_diff1": 0.795514, + "nauc_mrr_at_100_max": 0.47797, + "nauc_mrr_at_100_std": -0.532436, + "nauc_mrr_at_100_diff1": 0.795629, + "nauc_mrr_at_1000_max": 0.477937, + "nauc_mrr_at_1000_std": -0.532397, + "nauc_mrr_at_1000_diff1": 0.795624, + "hit_rate_at_1": 0.77634, + "hit_rate_at_3": 0.89677, + "hit_rate_at_5": 0.92581, + "hit_rate_at_10": 0.94516, + "hit_rate_at_20": 0.96452, + "hit_rate_at_100": 0.99892, + "hit_rate_at_1000": 1.0, + "main_score": 0.85167, + "hf_subset": "zh", + "languages": [ + "zho-Hans" + ] + }, + { + "ndcg_at_1": 0.68037, + "ndcg_at_3": 0.75664, + "ndcg_at_5": 0.78057, + "ndcg_at_10": 0.79587, + "ndcg_at_20": 0.80538, + "ndcg_at_100": 0.81364, + "ndcg_at_1000": 0.8148, + "map_at_1": 0.65373, + "map_at_3": 0.73067, + "map_at_5": 0.74517, + "map_at_10": 0.75209, + "map_at_20": 0.75514, + "map_at_100": 0.75656, + "map_at_1000": 0.75662, + "recall_at_1": 0.65373, + "recall_at_3": 0.81475, + "recall_at_5": 0.87059, + "recall_at_10": 0.91598, + "recall_at_20": 0.95118, + "recall_at_100": 0.99178, + "recall_at_1000": 1.0, + "accuracy": 0.65373, + "precision_at_1": 0.68037, + "precision_at_3": 0.2888, + "precision_at_5": 0.18664, + "precision_at_10": 0.09887, + "precision_at_20": 0.0517, + "precision_at_100": 0.01084, + "precision_at_1000": 0.00109, + "mrr_at_1": 0.68037, + "mrr_at_3": 0.748373, + "mrr_at_5": 0.760243, + "mrr_at_10": 0.765563, + "mrr_at_20": 0.767672, + "mrr_at_100": 0.768726, + "mrr_at_1000": 0.76877, + "nauc_ndcg_at_1_max": 0.470632, + "nauc_ndcg_at_1_std": -0.239195, + "nauc_ndcg_at_1_diff1": 0.809813, + "nauc_ndcg_at_3_max": 0.465454, + "nauc_ndcg_at_3_std": -0.303133, + "nauc_ndcg_at_3_diff1": 0.77232, + "nauc_ndcg_at_5_max": 0.474517, + "nauc_ndcg_at_5_std": -0.299269, + "nauc_ndcg_at_5_diff1": 0.774982, + "nauc_ndcg_at_10_max": 0.481796, + "nauc_ndcg_at_10_std": -0.305616, + "nauc_ndcg_at_10_diff1": 0.777943, + "nauc_ndcg_at_20_max": 0.480087, + "nauc_ndcg_at_20_std": -0.300324, + "nauc_ndcg_at_20_diff1": 0.781442, + "nauc_ndcg_at_100_max": 0.477831, + "nauc_ndcg_at_100_std": -0.285242, + "nauc_ndcg_at_100_diff1": 0.783077, + "nauc_ndcg_at_1000_max": 0.476813, + "nauc_ndcg_at_1000_std": -0.287649, + "nauc_ndcg_at_1000_diff1": 0.783842, + "nauc_map_at_1_max": 0.425098, + "nauc_map_at_1_std": -0.263295, + "nauc_map_at_1_diff1": 0.802953, + "nauc_map_at_3_max": 0.452014, + "nauc_map_at_3_std": -0.299477, + "nauc_map_at_3_diff1": 0.781159, + "nauc_map_at_5_max": 0.460609, + "nauc_map_at_5_std": -0.297936, + "nauc_map_at_5_diff1": 0.782928, + "nauc_map_at_10_max": 0.464238, + "nauc_map_at_10_std": -0.298726, + "nauc_map_at_10_diff1": 0.784359, + "nauc_map_at_20_max": 0.464043, + "nauc_map_at_20_std": -0.296468, + "nauc_map_at_20_diff1": 0.785351, + "nauc_map_at_100_max": 0.463861, + "nauc_map_at_100_std": -0.294054, + "nauc_map_at_100_diff1": 0.785651, + "nauc_map_at_1000_max": 0.463819, + "nauc_map_at_1000_std": -0.294127, + "nauc_map_at_1000_diff1": 0.785664, + "nauc_recall_at_1_max": 0.425098, + "nauc_recall_at_1_std": -0.263295, + "nauc_recall_at_1_diff1": 0.802953, + "nauc_recall_at_3_max": 0.457963, + "nauc_recall_at_3_std": -0.344885, + "nauc_recall_at_3_diff1": 0.72026, + "nauc_recall_at_5_max": 0.483935, + "nauc_recall_at_5_std": -0.344816, + "nauc_recall_at_5_diff1": 0.716305, + "nauc_recall_at_10_max": 0.522304, + "nauc_recall_at_10_std": -0.419044, + "nauc_recall_at_10_diff1": 0.699773, + "nauc_recall_at_20_max": 0.535847, + "nauc_recall_at_20_std": -0.46081, + "nauc_recall_at_20_diff1": 0.716573, + "nauc_recall_at_100_max": 0.6192, + "nauc_recall_at_100_std": 0.074323, + "nauc_recall_at_100_diff1": 0.654902, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.470632, + "nauc_precision_at_1_std": -0.239195, + "nauc_precision_at_1_diff1": 0.809813, + "nauc_precision_at_3_max": 0.434385, + "nauc_precision_at_3_std": -0.206746, + "nauc_precision_at_3_diff1": 0.489866, + "nauc_precision_at_5_max": 0.396095, + "nauc_precision_at_5_std": -0.119421, + "nauc_precision_at_5_diff1": 0.324994, + "nauc_precision_at_10_max": 0.33007, + "nauc_precision_at_10_std": -0.044429, + "nauc_precision_at_10_diff1": 0.138355, + "nauc_precision_at_20_max": 0.227181, + "nauc_precision_at_20_std": 0.067249, + "nauc_precision_at_20_diff1": -0.022537, + "nauc_precision_at_100_max": 0.095104, + "nauc_precision_at_100_std": 0.247551, + "nauc_precision_at_100_diff1": -0.234338, + "nauc_precision_at_1000_max": 0.06442, + "nauc_precision_at_1000_std": 0.254708, + "nauc_precision_at_1000_diff1": -0.278117, + "nauc_mrr_at_1_max": 0.470632, + "nauc_mrr_at_1_std": -0.239195, + "nauc_mrr_at_1_diff1": 0.809813, + "nauc_mrr_at_3_max": 0.492107, + "nauc_mrr_at_3_std": -0.269254, + "nauc_mrr_at_3_diff1": 0.784274, + "nauc_mrr_at_5_max": 0.490263, + "nauc_mrr_at_5_std": -0.265206, + "nauc_mrr_at_5_diff1": 0.785927, + "nauc_mrr_at_10_max": 0.491412, + "nauc_mrr_at_10_std": -0.268568, + "nauc_mrr_at_10_diff1": 0.787613, + "nauc_mrr_at_20_max": 0.490533, + "nauc_mrr_at_20_std": -0.268208, + "nauc_mrr_at_20_diff1": 0.788093, + "nauc_mrr_at_100_max": 0.490094, + "nauc_mrr_at_100_std": -0.266651, + "nauc_mrr_at_100_diff1": 0.788254, + "nauc_mrr_at_1000_max": 0.490039, + "nauc_mrr_at_1000_std": -0.266773, + "nauc_mrr_at_1000_diff1": 0.78828, + "hit_rate_at_1": 0.68037, + "hit_rate_at_3": 0.83556, + "hit_rate_at_5": 0.88798, + "hit_rate_at_10": 0.92806, + "hit_rate_at_20": 0.95786, + "hit_rate_at_100": 0.99281, + "hit_rate_at_1000": 1.0, + "main_score": 0.79587, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.70991, + "ndcg_at_3": 0.79714, + "ndcg_at_5": 0.81498, + "ndcg_at_10": 0.82998, + "ndcg_at_20": 0.8388, + "ndcg_at_100": 0.84307, + "ndcg_at_1000": 0.84325, + "map_at_1": 0.69101, + "map_at_3": 0.77105, + "map_at_5": 0.78146, + "map_at_10": 0.78851, + "map_at_20": 0.79141, + "map_at_100": 0.79216, + "map_at_1000": 0.79217, + "recall_at_1": 0.69101, + "recall_at_3": 0.85972, + "recall_at_5": 0.9016, + "recall_at_10": 0.94518, + "recall_at_20": 0.97804, + "recall_at_100": 0.99898, + "recall_at_1000": 1.0, + "accuracy": 0.69101, + "precision_at_1": 0.70991, + "precision_at_3": 0.30099, + "precision_at_5": 0.1904, + "precision_at_10": 0.10051, + "precision_at_20": 0.05225, + "precision_at_100": 0.0107, + "precision_at_1000": 0.00107, + "mrr_at_1": 0.709908, + "mrr_at_3": 0.783793, + "mrr_at_5": 0.792884, + "mrr_at_10": 0.798287, + "mrr_at_20": 0.800169, + "mrr_at_100": 0.80069, + "mrr_at_1000": 0.80069, + "nauc_ndcg_at_1_max": 0.356889, + "nauc_ndcg_at_1_std": -0.199916, + "nauc_ndcg_at_1_diff1": 0.760241, + "nauc_ndcg_at_3_max": 0.382347, + "nauc_ndcg_at_3_std": -0.247039, + "nauc_ndcg_at_3_diff1": 0.73203, + "nauc_ndcg_at_5_max": 0.398259, + "nauc_ndcg_at_5_std": -0.249338, + "nauc_ndcg_at_5_diff1": 0.727646, + "nauc_ndcg_at_10_max": 0.400459, + "nauc_ndcg_at_10_std": -0.236618, + "nauc_ndcg_at_10_diff1": 0.730736, + "nauc_ndcg_at_20_max": 0.387335, + "nauc_ndcg_at_20_std": -0.235105, + "nauc_ndcg_at_20_diff1": 0.732488, + "nauc_ndcg_at_100_max": 0.384046, + "nauc_ndcg_at_100_std": -0.230046, + "nauc_ndcg_at_100_diff1": 0.735304, + "nauc_ndcg_at_1000_max": 0.383442, + "nauc_ndcg_at_1000_std": -0.230465, + "nauc_ndcg_at_1000_diff1": 0.735357, + "nauc_map_at_1_max": 0.3386, + "nauc_map_at_1_std": -0.203943, + "nauc_map_at_1_diff1": 0.751666, + "nauc_map_at_3_max": 0.373124, + "nauc_map_at_3_std": -0.233932, + "nauc_map_at_3_diff1": 0.736939, + "nauc_map_at_5_max": 0.38059, + "nauc_map_at_5_std": -0.236629, + "nauc_map_at_5_diff1": 0.734597, + "nauc_map_at_10_max": 0.380939, + "nauc_map_at_10_std": -0.232425, + "nauc_map_at_10_diff1": 0.735598, + "nauc_map_at_20_max": 0.37776, + "nauc_map_at_20_std": -0.231299, + "nauc_map_at_20_diff1": 0.736075, + "nauc_map_at_100_max": 0.37731, + "nauc_map_at_100_std": -0.230431, + "nauc_map_at_100_diff1": 0.736564, + "nauc_map_at_1000_max": 0.377269, + "nauc_map_at_1000_std": -0.230461, + "nauc_map_at_1000_diff1": 0.736563, + "nauc_recall_at_1_max": 0.3386, + "nauc_recall_at_1_std": -0.203943, + "nauc_recall_at_1_diff1": 0.751666, + "nauc_recall_at_3_max": 0.410335, + "nauc_recall_at_3_std": -0.304998, + "nauc_recall_at_3_diff1": 0.702695, + "nauc_recall_at_5_max": 0.485066, + "nauc_recall_at_5_std": -0.328082, + "nauc_recall_at_5_diff1": 0.670685, + "nauc_recall_at_10_max": 0.570117, + "nauc_recall_at_10_std": -0.290733, + "nauc_recall_at_10_diff1": 0.662759, + "nauc_recall_at_20_max": 0.501868, + "nauc_recall_at_20_std": -0.359643, + "nauc_recall_at_20_diff1": 0.629547, + "nauc_recall_at_100_max": 0.861085, + "nauc_recall_at_100_std": 0.093141, + "nauc_recall_at_100_diff1": 0.678908, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.356889, + "nauc_precision_at_1_std": -0.199916, + "nauc_precision_at_1_diff1": 0.760241, + "nauc_precision_at_3_max": 0.340498, + "nauc_precision_at_3_std": -0.194503, + "nauc_precision_at_3_diff1": 0.448826, + "nauc_precision_at_5_max": 0.32492, + "nauc_precision_at_5_std": -0.158132, + "nauc_precision_at_5_diff1": 0.277882, + "nauc_precision_at_10_max": 0.224519, + "nauc_precision_at_10_std": -0.010912, + "nauc_precision_at_10_diff1": 0.042205, + "nauc_precision_at_20_max": 0.055003, + "nauc_precision_at_20_std": 0.088609, + "nauc_precision_at_20_diff1": -0.1695, + "nauc_precision_at_100_max": -0.048974, + "nauc_precision_at_100_std": 0.172383, + "nauc_precision_at_100_diff1": -0.309356, + "nauc_precision_at_1000_max": -0.061546, + "nauc_precision_at_1000_std": 0.166018, + "nauc_precision_at_1000_diff1": -0.311642, + "nauc_mrr_at_1_max": 0.356889, + "nauc_mrr_at_1_std": -0.199916, + "nauc_mrr_at_1_diff1": 0.760241, + "nauc_mrr_at_3_max": 0.377098, + "nauc_mrr_at_3_std": -0.231397, + "nauc_mrr_at_3_diff1": 0.74012, + "nauc_mrr_at_5_max": 0.38411, + "nauc_mrr_at_5_std": -0.228818, + "nauc_mrr_at_5_diff1": 0.739691, + "nauc_mrr_at_10_max": 0.384402, + "nauc_mrr_at_10_std": -0.22352, + "nauc_mrr_at_10_diff1": 0.740715, + "nauc_mrr_at_20_max": 0.380845, + "nauc_mrr_at_20_std": -0.224947, + "nauc_mrr_at_20_diff1": 0.740976, + "nauc_mrr_at_100_max": 0.380621, + "nauc_mrr_at_100_std": -0.224544, + "nauc_mrr_at_100_diff1": 0.741507, + "nauc_mrr_at_1000_max": 0.380621, + "nauc_mrr_at_1000_std": -0.224544, + "nauc_mrr_at_1000_diff1": 0.741507, + "hit_rate_at_1": 0.70991, + "hit_rate_at_3": 0.8764, + "hit_rate_at_5": 0.91624, + "hit_rate_at_10": 0.95608, + "hit_rate_at_20": 0.98264, + "hit_rate_at_100": 1.0, + "hit_rate_at_1000": 1.0, + "main_score": 0.82998, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.70763, + "ndcg_at_3": 0.77855, + "ndcg_at_5": 0.79835, + "ndcg_at_10": 0.81245, + "ndcg_at_20": 0.82084, + "ndcg_at_100": 0.82801, + "ndcg_at_1000": 0.82994, + "map_at_1": 0.68503, + "map_at_3": 0.75895, + "map_at_5": 0.77052, + "map_at_10": 0.77672, + "map_at_20": 0.77931, + "map_at_100": 0.78028, + "map_at_1000": 0.78037, + "recall_at_1": 0.68503, + "recall_at_3": 0.82786, + "recall_at_5": 0.87465, + "recall_at_10": 0.91649, + "recall_at_20": 0.94809, + "recall_at_100": 0.98614, + "recall_at_1000": 1.0, + "accuracy": 0.68503, + "precision_at_1": 0.70763, + "precision_at_3": 0.29131, + "precision_at_5": 0.18559, + "precision_at_10": 0.09746, + "precision_at_20": 0.05053, + "precision_at_100": 0.01057, + "precision_at_1000": 0.00107, + "mrr_at_1": 0.707627, + "mrr_at_3": 0.766243, + "mrr_at_5": 0.776254, + "mrr_at_10": 0.782002, + "mrr_at_20": 0.784104, + "mrr_at_100": 0.784949, + "mrr_at_1000": 0.785028, + "nauc_ndcg_at_1_max": 0.451615, + "nauc_ndcg_at_1_std": -0.259672, + "nauc_ndcg_at_1_diff1": 0.78114, + "nauc_ndcg_at_3_max": 0.481302, + "nauc_ndcg_at_3_std": -0.327107, + "nauc_ndcg_at_3_diff1": 0.761244, + "nauc_ndcg_at_5_max": 0.493703, + "nauc_ndcg_at_5_std": -0.341432, + "nauc_ndcg_at_5_diff1": 0.761835, + "nauc_ndcg_at_10_max": 0.487629, + "nauc_ndcg_at_10_std": -0.35878, + "nauc_ndcg_at_10_diff1": 0.760827, + "nauc_ndcg_at_20_max": 0.491792, + "nauc_ndcg_at_20_std": -0.340832, + "nauc_ndcg_at_20_diff1": 0.763584, + "nauc_ndcg_at_100_max": 0.486475, + "nauc_ndcg_at_100_std": -0.326618, + "nauc_ndcg_at_100_diff1": 0.766136, + "nauc_ndcg_at_1000_max": 0.48281, + "nauc_ndcg_at_1000_std": -0.325992, + "nauc_ndcg_at_1000_diff1": 0.764867, + "nauc_map_at_1_max": 0.393336, + "nauc_map_at_1_std": -0.250239, + "nauc_map_at_1_diff1": 0.767636, + "nauc_map_at_3_max": 0.467733, + "nauc_map_at_3_std": -0.306464, + "nauc_map_at_3_diff1": 0.76263, + "nauc_map_at_5_max": 0.475629, + "nauc_map_at_5_std": -0.312113, + "nauc_map_at_5_diff1": 0.763197, + "nauc_map_at_10_max": 0.473081, + "nauc_map_at_10_std": -0.318628, + "nauc_map_at_10_diff1": 0.762854, + "nauc_map_at_20_max": 0.474047, + "nauc_map_at_20_std": -0.313782, + "nauc_map_at_20_diff1": 0.763839, + "nauc_map_at_100_max": 0.473359, + "nauc_map_at_100_std": -0.312389, + "nauc_map_at_100_diff1": 0.764149, + "nauc_map_at_1000_max": 0.4732, + "nauc_map_at_1000_std": -0.312336, + "nauc_map_at_1000_diff1": 0.764087, + "nauc_recall_at_1_max": 0.393336, + "nauc_recall_at_1_std": -0.250239, + "nauc_recall_at_1_diff1": 0.767636, + "nauc_recall_at_3_max": 0.511114, + "nauc_recall_at_3_std": -0.400448, + "nauc_recall_at_3_diff1": 0.743305, + "nauc_recall_at_5_max": 0.560626, + "nauc_recall_at_5_std": -0.477803, + "nauc_recall_at_5_diff1": 0.743721, + "nauc_recall_at_10_max": 0.554654, + "nauc_recall_at_10_std": -0.633936, + "nauc_recall_at_10_diff1": 0.728448, + "nauc_recall_at_20_max": 0.646519, + "nauc_recall_at_20_std": -0.590909, + "nauc_recall_at_20_diff1": 0.744734, + "nauc_recall_at_100_max": 0.773788, + "nauc_recall_at_100_std": -0.367758, + "nauc_recall_at_100_diff1": 0.859438, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.451615, + "nauc_precision_at_1_std": -0.259672, + "nauc_precision_at_1_diff1": 0.78114, + "nauc_precision_at_3_max": 0.461008, + "nauc_precision_at_3_std": -0.274125, + "nauc_precision_at_3_diff1": 0.470565, + "nauc_precision_at_5_max": 0.434085, + "nauc_precision_at_5_std": -0.26597, + "nauc_precision_at_5_diff1": 0.348597, + "nauc_precision_at_10_max": 0.369072, + "nauc_precision_at_10_std": -0.285874, + "nauc_precision_at_10_diff1": 0.21354, + "nauc_precision_at_20_max": 0.307716, + "nauc_precision_at_20_std": -0.167868, + "nauc_precision_at_20_diff1": 0.07604, + "nauc_precision_at_100_max": 0.140432, + "nauc_precision_at_100_std": -0.001199, + "nauc_precision_at_100_diff1": -0.170795, + "nauc_precision_at_1000_max": 0.050865, + "nauc_precision_at_1000_std": 0.042988, + "nauc_precision_at_1000_diff1": -0.289924, + "nauc_mrr_at_1_max": 0.451615, + "nauc_mrr_at_1_std": -0.259672, + "nauc_mrr_at_1_diff1": 0.78114, + "nauc_mrr_at_3_max": 0.481819, + "nauc_mrr_at_3_std": -0.309886, + "nauc_mrr_at_3_diff1": 0.770648, + "nauc_mrr_at_5_max": 0.485733, + "nauc_mrr_at_5_std": -0.31715, + "nauc_mrr_at_5_diff1": 0.770644, + "nauc_mrr_at_10_max": 0.482316, + "nauc_mrr_at_10_std": -0.32302, + "nauc_mrr_at_10_diff1": 0.770273, + "nauc_mrr_at_20_max": 0.483582, + "nauc_mrr_at_20_std": -0.317845, + "nauc_mrr_at_20_diff1": 0.77079, + "nauc_mrr_at_100_max": 0.48304, + "nauc_mrr_at_100_std": -0.316061, + "nauc_mrr_at_100_diff1": 0.771202, + "nauc_mrr_at_1000_max": 0.482921, + "nauc_mrr_at_1000_std": -0.316019, + "nauc_mrr_at_1000_diff1": 0.771166, + "hit_rate_at_1": 0.70763, + "hit_rate_at_3": 0.83581, + "hit_rate_at_5": 0.8803, + "hit_rate_at_10": 0.92161, + "hit_rate_at_20": 0.95021, + "hit_rate_at_100": 0.98729, + "hit_rate_at_1000": 1.0, + "main_score": 0.81245, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ] + }, + { + "ndcg_at_1": 0.60125, + "ndcg_at_3": 0.67274, + "ndcg_at_5": 0.69293, + "ndcg_at_10": 0.71063, + "ndcg_at_20": 0.72413, + "ndcg_at_100": 0.73934, + "ndcg_at_1000": 0.7452, + "map_at_1": 0.58307, + "map_at_3": 0.65002, + "map_at_5": 0.66157, + "map_at_10": 0.6688, + "map_at_20": 0.6727, + "map_at_100": 0.67508, + "map_at_1000": 0.67537, + "recall_at_1": 0.58307, + "recall_at_3": 0.72672, + "recall_at_5": 0.77432, + "recall_at_10": 0.8297, + "recall_at_20": 0.88197, + "recall_at_100": 0.95916, + "recall_at_1000": 1.0, + "accuracy": 0.58307, + "precision_at_1": 0.60125, + "precision_at_3": 0.25303, + "precision_at_5": 0.16241, + "precision_at_10": 0.08733, + "precision_at_20": 0.04657, + "precision_at_100": 0.01024, + "precision_at_1000": 0.00108, + "mrr_at_1": 0.601246, + "mrr_at_3": 0.662513, + "mrr_at_5": 0.67352, + "mrr_at_10": 0.680121, + "mrr_at_20": 0.683489, + "mrr_at_100": 0.685495, + "mrr_at_1000": 0.685728, + "nauc_ndcg_at_1_max": 0.398323, + "nauc_ndcg_at_1_std": -0.270832, + "nauc_ndcg_at_1_diff1": 0.762474, + "nauc_ndcg_at_3_max": 0.421964, + "nauc_ndcg_at_3_std": -0.29553, + "nauc_ndcg_at_3_diff1": 0.713436, + "nauc_ndcg_at_5_max": 0.429593, + "nauc_ndcg_at_5_std": -0.307136, + "nauc_ndcg_at_5_diff1": 0.716906, + "nauc_ndcg_at_10_max": 0.438915, + "nauc_ndcg_at_10_std": -0.291196, + "nauc_ndcg_at_10_diff1": 0.720034, + "nauc_ndcg_at_20_max": 0.439668, + "nauc_ndcg_at_20_std": -0.294072, + "nauc_ndcg_at_20_diff1": 0.720034, + "nauc_ndcg_at_100_max": 0.427691, + "nauc_ndcg_at_100_std": -0.298695, + "nauc_ndcg_at_100_diff1": 0.72575, + "nauc_ndcg_at_1000_max": 0.42526, + "nauc_ndcg_at_1000_std": -0.293998, + "nauc_ndcg_at_1000_diff1": 0.726191, + "nauc_map_at_1_max": 0.378746, + "nauc_map_at_1_std": -0.256732, + "nauc_map_at_1_diff1": 0.747883, + "nauc_map_at_3_max": 0.409942, + "nauc_map_at_3_std": -0.285106, + "nauc_map_at_3_diff1": 0.722598, + "nauc_map_at_5_max": 0.414449, + "nauc_map_at_5_std": -0.291414, + "nauc_map_at_5_diff1": 0.725658, + "nauc_map_at_10_max": 0.417758, + "nauc_map_at_10_std": -0.285635, + "nauc_map_at_10_diff1": 0.726825, + "nauc_map_at_20_max": 0.417802, + "nauc_map_at_20_std": -0.286576, + "nauc_map_at_20_diff1": 0.727331, + "nauc_map_at_100_max": 0.415938, + "nauc_map_at_100_std": -0.28729, + "nauc_map_at_100_diff1": 0.727845, + "nauc_map_at_1000_max": 0.415856, + "nauc_map_at_1000_std": -0.287016, + "nauc_map_at_1000_diff1": 0.727885, + "nauc_recall_at_1_max": 0.378746, + "nauc_recall_at_1_std": -0.256732, + "nauc_recall_at_1_diff1": 0.747883, + "nauc_recall_at_3_max": 0.445479, + "nauc_recall_at_3_std": -0.317375, + "nauc_recall_at_3_diff1": 0.664478, + "nauc_recall_at_5_max": 0.469712, + "nauc_recall_at_5_std": -0.356289, + "nauc_recall_at_5_diff1": 0.665541, + "nauc_recall_at_10_max": 0.524427, + "nauc_recall_at_10_std": -0.288863, + "nauc_recall_at_10_diff1": 0.664179, + "nauc_recall_at_20_max": 0.56513, + "nauc_recall_at_20_std": -0.310798, + "nauc_recall_at_20_diff1": 0.643801, + "nauc_recall_at_100_max": 0.505927, + "nauc_recall_at_100_std": -0.454054, + "nauc_recall_at_100_diff1": 0.698283, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.398323, + "nauc_precision_at_1_std": -0.270832, + "nauc_precision_at_1_diff1": 0.762474, + "nauc_precision_at_3_max": 0.4375, + "nauc_precision_at_3_std": -0.304762, + "nauc_precision_at_3_diff1": 0.599431, + "nauc_precision_at_5_max": 0.437774, + "nauc_precision_at_5_std": -0.332747, + "nauc_precision_at_5_diff1": 0.570275, + "nauc_precision_at_10_max": 0.445049, + "nauc_precision_at_10_std": -0.255885, + "nauc_precision_at_10_diff1": 0.499895, + "nauc_precision_at_20_max": 0.399889, + "nauc_precision_at_20_std": -0.243798, + "nauc_precision_at_20_diff1": 0.386262, + "nauc_precision_at_100_max": 0.129695, + "nauc_precision_at_100_std": -0.14908, + "nauc_precision_at_100_diff1": 0.078892, + "nauc_precision_at_1000_max": -0.066062, + "nauc_precision_at_1000_std": -0.010944, + "nauc_precision_at_1000_diff1": -0.178522, + "nauc_mrr_at_1_max": 0.398323, + "nauc_mrr_at_1_std": -0.270832, + "nauc_mrr_at_1_diff1": 0.762474, + "nauc_mrr_at_3_max": 0.420125, + "nauc_mrr_at_3_std": -0.293652, + "nauc_mrr_at_3_diff1": 0.730649, + "nauc_mrr_at_5_max": 0.423868, + "nauc_mrr_at_5_std": -0.297872, + "nauc_mrr_at_5_diff1": 0.73164, + "nauc_mrr_at_10_max": 0.426068, + "nauc_mrr_at_10_std": -0.292387, + "nauc_mrr_at_10_diff1": 0.733101, + "nauc_mrr_at_20_max": 0.426316, + "nauc_mrr_at_20_std": -0.292289, + "nauc_mrr_at_20_diff1": 0.733067, + "nauc_mrr_at_100_max": 0.424485, + "nauc_mrr_at_100_std": -0.293173, + "nauc_mrr_at_100_diff1": 0.733953, + "nauc_mrr_at_1000_max": 0.424418, + "nauc_mrr_at_1000_std": -0.292869, + "nauc_mrr_at_1000_diff1": 0.733953, + "hit_rate_at_1": 0.60125, + "hit_rate_at_3": 0.74039, + "hit_rate_at_5": 0.78816, + "hit_rate_at_10": 0.84216, + "hit_rate_at_20": 0.892, + "hit_rate_at_100": 0.96469, + "hit_rate_at_1000": 1.0, + "main_score": 0.71063, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.77767, + "ndcg_at_3": 0.85585, + "ndcg_at_5": 0.87027, + "ndcg_at_10": 0.87848, + "ndcg_at_20": 0.88318, + "ndcg_at_100": 0.88543, + "ndcg_at_1000": 0.88571, + "map_at_1": 0.75637, + "map_at_3": 0.83389, + "map_at_5": 0.84289, + "map_at_10": 0.84667, + "map_at_20": 0.84818, + "map_at_100": 0.84855, + "map_at_1000": 0.84856, + "recall_at_1": 0.75637, + "recall_at_3": 0.91088, + "recall_at_5": 0.94492, + "recall_at_10": 0.96915, + "recall_at_20": 0.98667, + "recall_at_100": 0.99799, + "recall_at_1000": 1.0, + "accuracy": 0.75637, + "precision_at_1": 0.77767, + "precision_at_3": 0.31858, + "precision_at_5": 0.1992, + "precision_at_10": 0.10241, + "precision_at_20": 0.05226, + "precision_at_100": 0.01058, + "precision_at_1000": 0.00106, + "mrr_at_1": 0.777666, + "mrr_at_3": 0.841046, + "mrr_at_5": 0.84834, + "mrr_at_10": 0.851384, + "mrr_at_20": 0.852553, + "mrr_at_100": 0.852856, + "mrr_at_1000": 0.85287, + "nauc_ndcg_at_1_max": 0.430055, + "nauc_ndcg_at_1_std": -0.298711, + "nauc_ndcg_at_1_diff1": 0.817337, + "nauc_ndcg_at_3_max": 0.425957, + "nauc_ndcg_at_3_std": -0.378013, + "nauc_ndcg_at_3_diff1": 0.779313, + "nauc_ndcg_at_5_max": 0.43059, + "nauc_ndcg_at_5_std": -0.378829, + "nauc_ndcg_at_5_diff1": 0.791439, + "nauc_ndcg_at_10_max": 0.434619, + "nauc_ndcg_at_10_std": -0.368529, + "nauc_ndcg_at_10_diff1": 0.794761, + "nauc_ndcg_at_20_max": 0.434427, + "nauc_ndcg_at_20_std": -0.354481, + "nauc_ndcg_at_20_diff1": 0.796002, + "nauc_ndcg_at_100_max": 0.430911, + "nauc_ndcg_at_100_std": -0.354031, + "nauc_ndcg_at_100_diff1": 0.795269, + "nauc_ndcg_at_1000_max": 0.430849, + "nauc_ndcg_at_1000_std": -0.351899, + "nauc_ndcg_at_1000_diff1": 0.795264, + "nauc_map_at_1_max": 0.397718, + "nauc_map_at_1_std": -0.306194, + "nauc_map_at_1_diff1": 0.813012, + "nauc_map_at_3_max": 0.42274, + "nauc_map_at_3_std": -0.367091, + "nauc_map_at_3_diff1": 0.788836, + "nauc_map_at_5_max": 0.426798, + "nauc_map_at_5_std": -0.364138, + "nauc_map_at_5_diff1": 0.795847, + "nauc_map_at_10_max": 0.428111, + "nauc_map_at_10_std": -0.359924, + "nauc_map_at_10_diff1": 0.796776, + "nauc_map_at_20_max": 0.428243, + "nauc_map_at_20_std": -0.355132, + "nauc_map_at_20_diff1": 0.797061, + "nauc_map_at_100_max": 0.427857, + "nauc_map_at_100_std": -0.354755, + "nauc_map_at_100_diff1": 0.796926, + "nauc_map_at_1000_max": 0.427855, + "nauc_map_at_1000_std": -0.35467, + "nauc_map_at_1000_diff1": 0.796925, + "nauc_recall_at_1_max": 0.397718, + "nauc_recall_at_1_std": -0.306194, + "nauc_recall_at_1_diff1": 0.813012, + "nauc_recall_at_3_max": 0.422871, + "nauc_recall_at_3_std": -0.477046, + "nauc_recall_at_3_diff1": 0.723636, + "nauc_recall_at_5_max": 0.443026, + "nauc_recall_at_5_std": -0.539717, + "nauc_recall_at_5_diff1": 0.754805, + "nauc_recall_at_10_max": 0.500626, + "nauc_recall_at_10_std": -0.560832, + "nauc_recall_at_10_diff1": 0.773327, + "nauc_recall_at_20_max": 0.580395, + "nauc_recall_at_20_std": -0.452929, + "nauc_recall_at_20_diff1": 0.811068, + "nauc_recall_at_100_max": 0.45612, + "nauc_recall_at_100_std": -1.15265, + "nauc_recall_at_100_diff1": 0.795792, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.430055, + "nauc_precision_at_1_std": -0.298711, + "nauc_precision_at_1_diff1": 0.817337, + "nauc_precision_at_3_max": 0.263324, + "nauc_precision_at_3_std": -0.206831, + "nauc_precision_at_3_diff1": 0.268727, + "nauc_precision_at_5_max": 0.191541, + "nauc_precision_at_5_std": -0.096343, + "nauc_precision_at_5_diff1": 0.096638, + "nauc_precision_at_10_max": 0.123636, + "nauc_precision_at_10_std": 0.034481, + "nauc_precision_at_10_diff1": -0.069908, + "nauc_precision_at_20_max": 0.058779, + "nauc_precision_at_20_std": 0.184605, + "nauc_precision_at_20_diff1": -0.214732, + "nauc_precision_at_100_max": -0.008715, + "nauc_precision_at_100_std": 0.24265, + "nauc_precision_at_100_diff1": -0.323733, + "nauc_precision_at_1000_max": -0.017611, + "nauc_precision_at_1000_std": 0.269353, + "nauc_precision_at_1000_diff1": -0.345158, + "nauc_mrr_at_1_max": 0.430055, + "nauc_mrr_at_1_std": -0.298711, + "nauc_mrr_at_1_diff1": 0.817337, + "nauc_mrr_at_3_max": 0.430124, + "nauc_mrr_at_3_std": -0.34459, + "nauc_mrr_at_3_diff1": 0.789639, + "nauc_mrr_at_5_max": 0.429768, + "nauc_mrr_at_5_std": -0.342552, + "nauc_mrr_at_5_diff1": 0.794327, + "nauc_mrr_at_10_max": 0.431226, + "nauc_mrr_at_10_std": -0.340117, + "nauc_mrr_at_10_diff1": 0.7962, + "nauc_mrr_at_20_max": 0.431117, + "nauc_mrr_at_20_std": -0.338056, + "nauc_mrr_at_20_diff1": 0.796598, + "nauc_mrr_at_100_max": 0.430651, + "nauc_mrr_at_100_std": -0.338025, + "nauc_mrr_at_100_diff1": 0.796566, + "nauc_mrr_at_1000_max": 0.430649, + "nauc_mrr_at_1000_std": -0.337937, + "nauc_mrr_at_1000_diff1": 0.796566, + "hit_rate_at_1": 0.77767, + "hit_rate_at_3": 0.91851, + "hit_rate_at_5": 0.9497, + "hit_rate_at_10": 0.97183, + "hit_rate_at_20": 0.98793, + "hit_rate_at_100": 0.99799, + "hit_rate_at_1000": 1.0, + "main_score": 0.87848, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ] + }, + { + "ndcg_at_1": 0.67638, + "ndcg_at_3": 0.74971, + "ndcg_at_5": 0.7723, + "ndcg_at_10": 0.7896, + "ndcg_at_20": 0.7996, + "ndcg_at_100": 0.80853, + "ndcg_at_1000": 0.81046, + "map_at_1": 0.64846, + "map_at_3": 0.72673, + "map_at_5": 0.74091, + "map_at_10": 0.74875, + "map_at_20": 0.75177, + "map_at_100": 0.7533, + "map_at_1000": 0.7534, + "recall_at_1": 0.64846, + "recall_at_3": 0.80264, + "recall_at_5": 0.85397, + "recall_at_10": 0.90479, + "recall_at_20": 0.94207, + "recall_at_100": 0.9863, + "recall_at_1000": 1.0, + "accuracy": 0.64846, + "precision_at_1": 0.67638, + "precision_at_3": 0.28894, + "precision_at_5": 0.18647, + "precision_at_10": 0.09927, + "precision_at_20": 0.05187, + "precision_at_100": 0.01093, + "precision_at_1000": 0.00111, + "mrr_at_1": 0.676379, + "mrr_at_3": 0.739854, + "mrr_at_5": 0.750468, + "mrr_at_10": 0.75698, + "mrr_at_20": 0.759557, + "mrr_at_100": 0.760696, + "mrr_at_1000": 0.760777, + "nauc_ndcg_at_1_max": 0.414167, + "nauc_ndcg_at_1_std": -0.19502, + "nauc_ndcg_at_1_diff1": 0.746655, + "nauc_ndcg_at_3_max": 0.440035, + "nauc_ndcg_at_3_std": -0.200445, + "nauc_ndcg_at_3_diff1": 0.720622, + "nauc_ndcg_at_5_max": 0.451365, + "nauc_ndcg_at_5_std": -0.19637, + "nauc_ndcg_at_5_diff1": 0.727519, + "nauc_ndcg_at_10_max": 0.450133, + "nauc_ndcg_at_10_std": -0.193685, + "nauc_ndcg_at_10_diff1": 0.724621, + "nauc_ndcg_at_20_max": 0.447378, + "nauc_ndcg_at_20_std": -0.192946, + "nauc_ndcg_at_20_diff1": 0.726993, + "nauc_ndcg_at_100_max": 0.446673, + "nauc_ndcg_at_100_std": -0.189296, + "nauc_ndcg_at_100_diff1": 0.727663, + "nauc_ndcg_at_1000_max": 0.444503, + "nauc_ndcg_at_1000_std": -0.190946, + "nauc_ndcg_at_1000_diff1": 0.729036, + "nauc_map_at_1_max": 0.372691, + "nauc_map_at_1_std": -0.187587, + "nauc_map_at_1_diff1": 0.740856, + "nauc_map_at_3_max": 0.43303, + "nauc_map_at_3_std": -0.196885, + "nauc_map_at_3_diff1": 0.72757, + "nauc_map_at_5_max": 0.440438, + "nauc_map_at_5_std": -0.194099, + "nauc_map_at_5_diff1": 0.731678, + "nauc_map_at_10_max": 0.440476, + "nauc_map_at_10_std": -0.193151, + "nauc_map_at_10_diff1": 0.731264, + "nauc_map_at_20_max": 0.439646, + "nauc_map_at_20_std": -0.192871, + "nauc_map_at_20_diff1": 0.731937, + "nauc_map_at_100_max": 0.439612, + "nauc_map_at_100_std": -0.192268, + "nauc_map_at_100_diff1": 0.73188, + "nauc_map_at_1000_max": 0.439527, + "nauc_map_at_1000_std": -0.192345, + "nauc_map_at_1000_diff1": 0.731936, + "nauc_recall_at_1_max": 0.372691, + "nauc_recall_at_1_std": -0.187587, + "nauc_recall_at_1_diff1": 0.740856, + "nauc_recall_at_3_max": 0.451602, + "nauc_recall_at_3_std": -0.219459, + "nauc_recall_at_3_diff1": 0.687461, + "nauc_recall_at_5_max": 0.490585, + "nauc_recall_at_5_std": -0.210736, + "nauc_recall_at_5_diff1": 0.701994, + "nauc_recall_at_10_max": 0.49706, + "nauc_recall_at_10_std": -0.208807, + "nauc_recall_at_10_diff1": 0.672422, + "nauc_recall_at_20_max": 0.491822, + "nauc_recall_at_20_std": -0.216225, + "nauc_recall_at_20_diff1": 0.667871, + "nauc_recall_at_100_max": 0.634928, + "nauc_recall_at_100_std": -0.056098, + "nauc_recall_at_100_diff1": 0.609427, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.414167, + "nauc_precision_at_1_std": -0.19502, + "nauc_precision_at_1_diff1": 0.746655, + "nauc_precision_at_3_max": 0.382713, + "nauc_precision_at_3_std": -0.117983, + "nauc_precision_at_3_diff1": 0.388812, + "nauc_precision_at_5_max": 0.322619, + "nauc_precision_at_5_std": -0.078017, + "nauc_precision_at_5_diff1": 0.253027, + "nauc_precision_at_10_max": 0.249087, + "nauc_precision_at_10_std": -0.021507, + "nauc_precision_at_10_diff1": 0.100768, + "nauc_precision_at_20_max": 0.166728, + "nauc_precision_at_20_std": 0.024075, + "nauc_precision_at_20_diff1": -0.037461, + "nauc_precision_at_100_max": 0.050285, + "nauc_precision_at_100_std": 0.106214, + "nauc_precision_at_100_diff1": -0.237917, + "nauc_precision_at_1000_max": -0.003756, + "nauc_precision_at_1000_std": 0.10491, + "nauc_precision_at_1000_diff1": -0.298842, + "nauc_mrr_at_1_max": 0.414167, + "nauc_mrr_at_1_std": -0.19502, + "nauc_mrr_at_1_diff1": 0.746655, + "nauc_mrr_at_3_max": 0.438482, + "nauc_mrr_at_3_std": -0.19356, + "nauc_mrr_at_3_diff1": 0.728833, + "nauc_mrr_at_5_max": 0.441554, + "nauc_mrr_at_5_std": -0.19104, + "nauc_mrr_at_5_diff1": 0.732488, + "nauc_mrr_at_10_max": 0.440296, + "nauc_mrr_at_10_std": -0.190015, + "nauc_mrr_at_10_diff1": 0.731212, + "nauc_mrr_at_20_max": 0.440035, + "nauc_mrr_at_20_std": -0.189638, + "nauc_mrr_at_20_diff1": 0.732039, + "nauc_mrr_at_100_max": 0.439704, + "nauc_mrr_at_100_std": -0.1901, + "nauc_mrr_at_100_diff1": 0.732266, + "nauc_mrr_at_1000_max": 0.439666, + "nauc_mrr_at_1000_std": -0.190073, + "nauc_mrr_at_1000_diff1": 0.732327, + "hit_rate_at_1": 0.67638, + "hit_rate_at_3": 0.81998, + "hit_rate_at_5": 0.86576, + "hit_rate_at_10": 0.91363, + "hit_rate_at_20": 0.94901, + "hit_rate_at_100": 0.98751, + "hit_rate_at_1000": 1.0, + "main_score": 0.7896, + "hf_subset": "sw", + "languages": [ + "swa-Latn" + ] + }, + { + "ndcg_at_1": 0.76222, + "ndcg_at_3": 0.83099, + "ndcg_at_5": 0.84788, + "ndcg_at_10": 0.86029, + "ndcg_at_20": 0.86607, + "ndcg_at_100": 0.87035, + "ndcg_at_1000": 0.87035, + "map_at_1": 0.74352, + "map_at_3": 0.8112, + "map_at_5": 0.82146, + "map_at_10": 0.82714, + "map_at_20": 0.82881, + "map_at_100": 0.82951, + "map_at_1000": 0.82951, + "recall_at_1": 0.74352, + "recall_at_3": 0.88, + "recall_at_5": 0.91926, + "recall_at_10": 0.9563, + "recall_at_20": 0.97852, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "accuracy": 0.74352, + "precision_at_1": 0.76222, + "precision_at_3": 0.30667, + "precision_at_5": 0.19311, + "precision_at_10": 0.10078, + "precision_at_20": 0.0515, + "precision_at_100": 0.01053, + "precision_at_1000": 0.00105, + "mrr_at_1": 0.762222, + "mrr_at_3": 0.819259, + "mrr_at_5": 0.827537, + "mrr_at_10": 0.831892, + "mrr_at_20": 0.833564, + "mrr_at_100": 0.834216, + "mrr_at_1000": 0.834216, + "nauc_ndcg_at_1_max": 0.326149, + "nauc_ndcg_at_1_std": -0.289058, + "nauc_ndcg_at_1_diff1": 0.791298, + "nauc_ndcg_at_3_max": 0.376583, + "nauc_ndcg_at_3_std": -0.288773, + "nauc_ndcg_at_3_diff1": 0.773765, + "nauc_ndcg_at_5_max": 0.38646, + "nauc_ndcg_at_5_std": -0.303623, + "nauc_ndcg_at_5_diff1": 0.779489, + "nauc_ndcg_at_10_max": 0.388632, + "nauc_ndcg_at_10_std": -0.298648, + "nauc_ndcg_at_10_diff1": 0.781482, + "nauc_ndcg_at_20_max": 0.382467, + "nauc_ndcg_at_20_std": -0.29938, + "nauc_ndcg_at_20_diff1": 0.783023, + "nauc_ndcg_at_100_max": 0.371223, + "nauc_ndcg_at_100_std": -0.293062, + "nauc_ndcg_at_100_diff1": 0.781322, + "nauc_ndcg_at_1000_max": 0.371223, + "nauc_ndcg_at_1000_std": -0.293062, + "nauc_ndcg_at_1000_diff1": 0.781322, + "nauc_map_at_1_max": 0.308487, + "nauc_map_at_1_std": -0.283091, + "nauc_map_at_1_diff1": 0.782348, + "nauc_map_at_3_max": 0.362471, + "nauc_map_at_3_std": -0.287606, + "nauc_map_at_3_diff1": 0.778804, + "nauc_map_at_5_max": 0.367892, + "nauc_map_at_5_std": -0.293462, + "nauc_map_at_5_diff1": 0.782277, + "nauc_map_at_10_max": 0.368164, + "nauc_map_at_10_std": -0.291544, + "nauc_map_at_10_diff1": 0.782211, + "nauc_map_at_20_max": 0.366257, + "nauc_map_at_20_std": -0.291864, + "nauc_map_at_20_diff1": 0.78253, + "nauc_map_at_100_max": 0.364931, + "nauc_map_at_100_std": -0.290922, + "nauc_map_at_100_diff1": 0.782319, + "nauc_map_at_1000_max": 0.364931, + "nauc_map_at_1000_std": -0.290922, + "nauc_map_at_1000_diff1": 0.782319, + "nauc_recall_at_1_max": 0.308487, + "nauc_recall_at_1_std": -0.283091, + "nauc_recall_at_1_diff1": 0.782348, + "nauc_recall_at_3_max": 0.42826, + "nauc_recall_at_3_std": -0.291603, + "nauc_recall_at_3_diff1": 0.750494, + "nauc_recall_at_5_max": 0.490217, + "nauc_recall_at_5_std": -0.360665, + "nauc_recall_at_5_diff1": 0.760871, + "nauc_recall_at_10_max": 0.602112, + "nauc_recall_at_10_std": -0.374956, + "nauc_recall_at_10_diff1": 0.780613, + "nauc_recall_at_20_max": 0.702095, + "nauc_recall_at_20_std": -0.456675, + "nauc_recall_at_20_diff1": 0.815855, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.326149, + "nauc_precision_at_1_std": -0.289058, + "nauc_precision_at_1_diff1": 0.791298, + "nauc_precision_at_3_max": 0.323777, + "nauc_precision_at_3_std": -0.169389, + "nauc_precision_at_3_diff1": 0.414045, + "nauc_precision_at_5_max": 0.283911, + "nauc_precision_at_5_std": -0.139967, + "nauc_precision_at_5_diff1": 0.249051, + "nauc_precision_at_10_max": 0.201115, + "nauc_precision_at_10_std": -0.055208, + "nauc_precision_at_10_diff1": 0.030324, + "nauc_precision_at_20_max": 0.137906, + "nauc_precision_at_20_std": -0.005207, + "nauc_precision_at_20_diff1": -0.118427, + "nauc_precision_at_100_max": -0.008323, + "nauc_precision_at_100_std": 0.118812, + "nauc_precision_at_100_diff1": -0.339136, + "nauc_precision_at_1000_max": -0.008323, + "nauc_precision_at_1000_std": 0.118812, + "nauc_precision_at_1000_diff1": -0.339136, + "nauc_mrr_at_1_max": 0.326149, + "nauc_mrr_at_1_std": -0.289058, + "nauc_mrr_at_1_diff1": 0.791298, + "nauc_mrr_at_3_max": 0.364251, + "nauc_mrr_at_3_std": -0.288494, + "nauc_mrr_at_3_diff1": 0.778761, + "nauc_mrr_at_5_max": 0.365081, + "nauc_mrr_at_5_std": -0.296926, + "nauc_mrr_at_5_diff1": 0.781107, + "nauc_mrr_at_10_max": 0.364842, + "nauc_mrr_at_10_std": -0.293152, + "nauc_mrr_at_10_diff1": 0.782788, + "nauc_mrr_at_20_max": 0.362856, + "nauc_mrr_at_20_std": -0.293497, + "nauc_mrr_at_20_diff1": 0.783121, + "nauc_mrr_at_100_max": 0.361489, + "nauc_mrr_at_100_std": -0.292743, + "nauc_mrr_at_100_diff1": 0.782933, + "nauc_mrr_at_1000_max": 0.361489, + "nauc_mrr_at_1000_std": -0.292743, + "nauc_mrr_at_1000_diff1": 0.782933, + "hit_rate_at_1": 0.76222, + "hit_rate_at_3": 0.89, + "hit_rate_at_5": 0.92556, + "hit_rate_at_10": 0.95778, + "hit_rate_at_20": 0.98, + "hit_rate_at_100": 1.0, + "hit_rate_at_1000": 1.0, + "main_score": 0.86029, + "hf_subset": "hi", + "languages": [ + "hin-Deva" + ] + }, + { + "ndcg_at_1": 0.71808, + "ndcg_at_3": 0.80391, + "ndcg_at_5": 0.82369, + "ndcg_at_10": 0.83735, + "ndcg_at_20": 0.84544, + "ndcg_at_100": 0.84979, + "ndcg_at_1000": 0.84994, + "map_at_1": 0.69203, + "map_at_3": 0.7784, + "map_at_5": 0.79047, + "map_at_10": 0.79698, + "map_at_20": 0.7995, + "map_at_100": 0.80032, + "map_at_1000": 0.80033, + "recall_at_1": 0.69203, + "recall_at_3": 0.86296, + "recall_at_5": 0.90841, + "recall_at_10": 0.94791, + "recall_at_20": 0.97838, + "recall_at_100": 0.99915, + "recall_at_1000": 1.0, + "accuracy": 0.69203, + "precision_at_1": 0.71808, + "precision_at_3": 0.30712, + "precision_at_5": 0.19571, + "precision_at_10": 0.10255, + "precision_at_20": 0.05312, + "precision_at_100": 0.0109, + "precision_at_1000": 0.00109, + "mrr_at_1": 0.71808, + "mrr_at_3": 0.792816, + "mrr_at_5": 0.802366, + "mrr_at_10": 0.807152, + "mrr_at_20": 0.808991, + "mrr_at_100": 0.809503, + "mrr_at_1000": 0.809503, + "nauc_ndcg_at_1_max": 0.359744, + "nauc_ndcg_at_1_std": -0.298852, + "nauc_ndcg_at_1_diff1": 0.784916, + "nauc_ndcg_at_3_max": 0.398499, + "nauc_ndcg_at_3_std": -0.392089, + "nauc_ndcg_at_3_diff1": 0.742036, + "nauc_ndcg_at_5_max": 0.407976, + "nauc_ndcg_at_5_std": -0.37215, + "nauc_ndcg_at_5_diff1": 0.741134, + "nauc_ndcg_at_10_max": 0.416865, + "nauc_ndcg_at_10_std": -0.359796, + "nauc_ndcg_at_10_diff1": 0.757728, + "nauc_ndcg_at_20_max": 0.414041, + "nauc_ndcg_at_20_std": -0.35652, + "nauc_ndcg_at_20_diff1": 0.755825, + "nauc_ndcg_at_100_max": 0.404508, + "nauc_ndcg_at_100_std": -0.349654, + "nauc_ndcg_at_100_diff1": 0.756976, + "nauc_ndcg_at_1000_max": 0.403882, + "nauc_ndcg_at_1000_std": -0.350646, + "nauc_ndcg_at_1000_diff1": 0.757139, + "nauc_map_at_1_max": 0.319148, + "nauc_map_at_1_std": -0.300706, + "nauc_map_at_1_diff1": 0.780102, + "nauc_map_at_3_max": 0.383775, + "nauc_map_at_3_std": -0.369722, + "nauc_map_at_3_diff1": 0.750841, + "nauc_map_at_5_max": 0.390276, + "nauc_map_at_5_std": -0.358215, + "nauc_map_at_5_diff1": 0.751167, + "nauc_map_at_10_max": 0.395578, + "nauc_map_at_10_std": -0.35333, + "nauc_map_at_10_diff1": 0.75726, + "nauc_map_at_20_max": 0.394828, + "nauc_map_at_20_std": -0.352034, + "nauc_map_at_20_diff1": 0.757046, + "nauc_map_at_100_max": 0.393417, + "nauc_map_at_100_std": -0.351062, + "nauc_map_at_100_diff1": 0.757271, + "nauc_map_at_1000_max": 0.393374, + "nauc_map_at_1000_std": -0.351124, + "nauc_map_at_1000_diff1": 0.757276, + "nauc_recall_at_1_max": 0.319148, + "nauc_recall_at_1_std": -0.300706, + "nauc_recall_at_1_diff1": 0.780102, + "nauc_recall_at_3_max": 0.423369, + "nauc_recall_at_3_std": -0.504078, + "nauc_recall_at_3_diff1": 0.691264, + "nauc_recall_at_5_max": 0.459358, + "nauc_recall_at_5_std": -0.478097, + "nauc_recall_at_5_diff1": 0.660978, + "nauc_recall_at_10_max": 0.560596, + "nauc_recall_at_10_std": -0.455078, + "nauc_recall_at_10_diff1": 0.748337, + "nauc_recall_at_20_max": 0.705044, + "nauc_recall_at_20_std": -0.514316, + "nauc_recall_at_20_diff1": 0.699958, + "nauc_recall_at_100_max": 0.947706, + "nauc_recall_at_100_std": 0.554029, + "nauc_recall_at_100_diff1": 0.576429, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.359744, + "nauc_precision_at_1_std": -0.298852, + "nauc_precision_at_1_diff1": 0.784916, + "nauc_precision_at_3_max": 0.380429, + "nauc_precision_at_3_std": -0.30469, + "nauc_precision_at_3_diff1": 0.33556, + "nauc_precision_at_5_max": 0.334269, + "nauc_precision_at_5_std": -0.138075, + "nauc_precision_at_5_diff1": 0.137907, + "nauc_precision_at_10_max": 0.289951, + "nauc_precision_at_10_std": -0.010015, + "nauc_precision_at_10_diff1": -0.007846, + "nauc_precision_at_20_max": 0.208726, + "nauc_precision_at_20_std": 0.100663, + "nauc_precision_at_20_diff1": -0.190212, + "nauc_precision_at_100_max": 0.085406, + "nauc_precision_at_100_std": 0.199341, + "nauc_precision_at_100_diff1": -0.304097, + "nauc_precision_at_1000_max": 0.071562, + "nauc_precision_at_1000_std": 0.18783, + "nauc_precision_at_1000_diff1": -0.304242, + "nauc_mrr_at_1_max": 0.359744, + "nauc_mrr_at_1_std": -0.298852, + "nauc_mrr_at_1_diff1": 0.784916, + "nauc_mrr_at_3_max": 0.404634, + "nauc_mrr_at_3_std": -0.351213, + "nauc_mrr_at_3_diff1": 0.758138, + "nauc_mrr_at_5_max": 0.405231, + "nauc_mrr_at_5_std": -0.341071, + "nauc_mrr_at_5_diff1": 0.7572, + "nauc_mrr_at_10_max": 0.403258, + "nauc_mrr_at_10_std": -0.336439, + "nauc_mrr_at_10_diff1": 0.76332, + "nauc_mrr_at_20_max": 0.401606, + "nauc_mrr_at_20_std": -0.337239, + "nauc_mrr_at_20_diff1": 0.762553, + "nauc_mrr_at_100_max": 0.400672, + "nauc_mrr_at_100_std": -0.336303, + "nauc_mrr_at_100_diff1": 0.762697, + "nauc_mrr_at_1000_max": 0.400672, + "nauc_mrr_at_1000_std": -0.336303, + "nauc_mrr_at_1000_diff1": 0.762697, + "hit_rate_at_1": 0.71808, + "hit_rate_at_3": 0.88151, + "hit_rate_at_5": 0.92339, + "hit_rate_at_10": 0.9571, + "hit_rate_at_20": 0.98366, + "hit_rate_at_100": 1.0, + "hit_rate_at_1000": 1.0, + "main_score": 0.83735, + "hf_subset": "ur", + "languages": [ + "urd-Arab" + ] + } + ] + }, + "evaluation_time": 315.96304631233215, + "kg_co2_emissions": null, + "date": 1777842717.082412 +} \ No newline at end of file diff --git a/results/BAAI__bge-m3/5617a9f61b028005a4858fdac845db406aefb181/LexRetrieval.v1.json b/results/BAAI__bge-m3/5617a9f61b028005a4858fdac845db406aefb181/LexRetrieval.v1.json new file mode 100644 index 0000000000..d74f4a2b3e --- /dev/null +++ b/results/BAAI__bge-m3/5617a9f61b028005a4858fdac845db406aefb181/LexRetrieval.v1.json @@ -0,0 +1,167 @@ +{ + "dataset_revision": "b5b22dee3a30bb32ac579c14afe9eb152e8c70c7", + "task_name": "LexRetrieval.v1", + "mteb_version": "2.12.38", + "scores": { + "test": [ + { + "ndcg_at_1": 0.92847, + "ndcg_at_3": 0.95093, + "ndcg_at_5": 0.95386, + "ndcg_at_10": 0.95586, + "ndcg_at_20": 0.95763, + "ndcg_at_100": 0.9592, + "ndcg_at_1000": 0.95986, + "map_at_1": 0.92847, + "map_at_3": 0.94577, + "map_at_5": 0.94738, + "map_at_10": 0.9482, + "map_at_20": 0.9487, + "map_at_100": 0.94892, + "map_at_1000": 0.94895, + "recall_at_1": 0.92847, + "recall_at_3": 0.96573, + "recall_at_5": 0.97288, + "recall_at_10": 0.97908, + "recall_at_20": 0.98601, + "recall_at_100": 0.99445, + "recall_at_1000": 0.99957, + "accuracy": 0.92847, + "precision_at_1": 0.92847, + "precision_at_3": 0.32191, + "precision_at_5": 0.19458, + "precision_at_10": 0.09791, + "precision_at_20": 0.0493, + "precision_at_100": 0.00994, + "precision_at_1000": 0.001, + "mrr_at_1": 0.928472, + "mrr_at_3": 0.945767, + "mrr_at_5": 0.947379, + "mrr_at_10": 0.948203, + "mrr_at_20": 0.948701, + "mrr_at_100": 0.94892, + "mrr_at_1000": 0.948947, + "nauc_ndcg_at_1_max": 0.781143, + "nauc_ndcg_at_1_std": 0.108718, + "nauc_ndcg_at_1_diff1": 0.938644, + "nauc_ndcg_at_3_max": 0.83664, + "nauc_ndcg_at_3_std": 0.141522, + "nauc_ndcg_at_3_diff1": 0.941949, + "nauc_ndcg_at_5_max": 0.834339, + "nauc_ndcg_at_5_std": 0.146197, + "nauc_ndcg_at_5_diff1": 0.943335, + "nauc_ndcg_at_10_max": 0.830838, + "nauc_ndcg_at_10_std": 0.150792, + "nauc_ndcg_at_10_diff1": 0.94225, + "nauc_ndcg_at_20_max": 0.828032, + "nauc_ndcg_at_20_std": 0.144516, + "nauc_ndcg_at_20_diff1": 0.941985, + "nauc_ndcg_at_100_max": 0.824269, + "nauc_ndcg_at_100_std": 0.145301, + "nauc_ndcg_at_100_diff1": 0.941242, + "nauc_ndcg_at_1000_max": 0.82254, + "nauc_ndcg_at_1000_std": 0.142244, + "nauc_ndcg_at_1000_diff1": 0.940784, + "nauc_map_at_1_max": 0.781143, + "nauc_map_at_1_std": 0.108718, + "nauc_map_at_1_diff1": 0.938644, + "nauc_map_at_3_max": 0.820983, + "nauc_map_at_3_std": 0.132523, + "nauc_map_at_3_diff1": 0.940719, + "nauc_map_at_5_max": 0.819332, + "nauc_map_at_5_std": 0.134488, + "nauc_map_at_5_diff1": 0.9413, + "nauc_map_at_10_max": 0.817824, + "nauc_map_at_10_std": 0.13606, + "nauc_map_at_10_diff1": 0.940899, + "nauc_map_at_20_max": 0.817073, + "nauc_map_at_20_std": 0.134248, + "nauc_map_at_20_diff1": 0.940832, + "nauc_map_at_100_max": 0.816612, + "nauc_map_at_100_std": 0.134646, + "nauc_map_at_100_diff1": 0.940737, + "nauc_map_at_1000_max": 0.816566, + "nauc_map_at_1000_std": 0.134587, + "nauc_map_at_1000_diff1": 0.940727, + "nauc_recall_at_1_max": 0.781143, + "nauc_recall_at_1_std": 0.108718, + "nauc_recall_at_1_diff1": 0.938644, + "nauc_recall_at_3_max": 0.906841, + "nauc_recall_at_3_std": 0.181683, + "nauc_recall_at_3_diff1": 0.947685, + "nauc_recall_at_5_max": 0.916274, + "nauc_recall_at_5_std": 0.211861, + "nauc_recall_at_5_diff1": 0.955376, + "nauc_recall_at_10_max": 0.918164, + "nauc_recall_at_10_std": 0.259405, + "nauc_recall_at_10_diff1": 0.951675, + "nauc_recall_at_20_max": 0.92748, + "nauc_recall_at_20_std": 0.24475, + "nauc_recall_at_20_diff1": 0.952979, + "nauc_recall_at_100_max": 0.929531, + "nauc_recall_at_100_std": 0.383832, + "nauc_recall_at_100_diff1": 0.940933, + "nauc_recall_at_1000_max": 0.855859, + "nauc_recall_at_1000_std": 0.715327, + "nauc_recall_at_1000_diff1": 0.57644, + "nauc_precision_at_1_max": 0.781143, + "nauc_precision_at_1_std": 0.108718, + "nauc_precision_at_1_diff1": 0.938644, + "nauc_precision_at_3_max": 0.906841, + "nauc_precision_at_3_std": 0.181683, + "nauc_precision_at_3_diff1": 0.947685, + "nauc_precision_at_5_max": 0.916274, + "nauc_precision_at_5_std": 0.211861, + "nauc_precision_at_5_diff1": 0.955376, + "nauc_precision_at_10_max": 0.918164, + "nauc_precision_at_10_std": 0.259405, + "nauc_precision_at_10_diff1": 0.951675, + "nauc_precision_at_20_max": 0.92748, + "nauc_precision_at_20_std": 0.24475, + "nauc_precision_at_20_diff1": 0.952979, + "nauc_precision_at_100_max": 0.929531, + "nauc_precision_at_100_std": 0.383832, + "nauc_precision_at_100_diff1": 0.940933, + "nauc_precision_at_1000_max": 0.855859, + "nauc_precision_at_1000_std": 0.715327, + "nauc_precision_at_1000_diff1": 0.57644, + "nauc_mrr_at_1_max": 0.781143, + "nauc_mrr_at_1_std": 0.108718, + "nauc_mrr_at_1_diff1": 0.938644, + "nauc_mrr_at_3_max": 0.820983, + "nauc_mrr_at_3_std": 0.132523, + "nauc_mrr_at_3_diff1": 0.940719, + "nauc_mrr_at_5_max": 0.819332, + "nauc_mrr_at_5_std": 0.134488, + "nauc_mrr_at_5_diff1": 0.9413, + "nauc_mrr_at_10_max": 0.817824, + "nauc_mrr_at_10_std": 0.13606, + "nauc_mrr_at_10_diff1": 0.940899, + "nauc_mrr_at_20_max": 0.817073, + "nauc_mrr_at_20_std": 0.134248, + "nauc_mrr_at_20_diff1": 0.940832, + "nauc_mrr_at_100_max": 0.816612, + "nauc_mrr_at_100_std": 0.134646, + "nauc_mrr_at_100_diff1": 0.940737, + "nauc_mrr_at_1000_max": 0.816566, + "nauc_mrr_at_1000_std": 0.134587, + "nauc_mrr_at_1000_diff1": 0.940727, + "hit_rate_at_1": 0.92847, + "hit_rate_at_3": 0.96573, + "hit_rate_at_5": 0.97288, + "hit_rate_at_10": 0.97908, + "hit_rate_at_20": 0.98601, + "hit_rate_at_100": 0.99445, + "hit_rate_at_1000": 0.99957, + "main_score": 0.95586, + "hf_subset": "default", + "languages": [ + "dan-Latn" + ] + } + ] + }, + "evaluation_time": 159.2563648223877, + "kg_co2_emissions": null, + "date": 1777842386.496898 +} \ No newline at end of file diff --git a/results/BAAI__bge-m3/5617a9f61b028005a4858fdac845db406aefb181/PublicNewsRetrieval.json b/results/BAAI__bge-m3/5617a9f61b028005a4858fdac845db406aefb181/PublicNewsRetrieval.json new file mode 100644 index 0000000000..df2bc77b60 --- /dev/null +++ b/results/BAAI__bge-m3/5617a9f61b028005a4858fdac845db406aefb181/PublicNewsRetrieval.json @@ -0,0 +1,322 @@ +{ + "dataset_revision": "82dad8c469646feae9631d69ba92bebfc0a586e9", + "task_name": "PublicNewsRetrieval", + "mteb_version": "2.12.37", + "scores": { + "test": [ + { + "ndcg_at_1": 0.76139, + "ndcg_at_3": 0.81771, + "ndcg_at_5": 0.83014, + "ndcg_at_10": 0.84091, + "ndcg_at_20": 0.84591, + "ndcg_at_100": 0.85305, + "ndcg_at_1000": 0.85598, + "map_at_1": 0.74955, + "map_at_3": 0.80148, + "map_at_5": 0.80875, + "map_at_10": 0.81356, + "map_at_20": 0.81506, + "map_at_100": 0.81615, + "map_at_1000": 0.81627, + "recall_at_1": 0.74955, + "recall_at_3": 0.85729, + "recall_at_5": 0.88729, + "recall_at_10": 0.91963, + "recall_at_20": 0.93851, + "recall_at_100": 0.97537, + "recall_at_1000": 0.99732, + "accuracy": 0.74955, + "precision_at_1": 0.76139, + "precision_at_3": 0.29424, + "precision_at_5": 0.18351, + "precision_at_10": 0.09538, + "precision_at_20": 0.04879, + "precision_at_100": 0.01017, + "precision_at_1000": 0.00104, + "mrr_at_1": 0.761394, + "mrr_at_3": 0.808199, + "mrr_at_5": 0.8146, + "mrr_at_10": 0.818666, + "mrr_at_20": 0.81993, + "mrr_at_100": 0.820908, + "mrr_at_1000": 0.821007, + "nauc_ndcg_at_1_max": 0.577069, + "nauc_ndcg_at_1_std": -0.227244, + "nauc_ndcg_at_1_diff1": 0.851721, + "nauc_ndcg_at_3_max": 0.634578, + "nauc_ndcg_at_3_std": -0.231898, + "nauc_ndcg_at_3_diff1": 0.831645, + "nauc_ndcg_at_5_max": 0.64322, + "nauc_ndcg_at_5_std": -0.224976, + "nauc_ndcg_at_5_diff1": 0.827878, + "nauc_ndcg_at_10_max": 0.645914, + "nauc_ndcg_at_10_std": -0.214816, + "nauc_ndcg_at_10_diff1": 0.83077, + "nauc_ndcg_at_20_max": 0.647848, + "nauc_ndcg_at_20_std": -0.20482, + "nauc_ndcg_at_20_diff1": 0.830811, + "nauc_ndcg_at_100_max": 0.641383, + "nauc_ndcg_at_100_std": -0.199098, + "nauc_ndcg_at_100_diff1": 0.83387, + "nauc_ndcg_at_1000_max": 0.636203, + "nauc_ndcg_at_1000_std": -0.208822, + "nauc_ndcg_at_1000_diff1": 0.834153, + "nauc_map_at_1_max": 0.554284, + "nauc_map_at_1_std": -0.251232, + "nauc_map_at_1_diff1": 0.853441, + "nauc_map_at_3_max": 0.616128, + "nauc_map_at_3_std": -0.239583, + "nauc_map_at_3_diff1": 0.836458, + "nauc_map_at_5_max": 0.62007, + "nauc_map_at_5_std": -0.234771, + "nauc_map_at_5_diff1": 0.834887, + "nauc_map_at_10_max": 0.621239, + "nauc_map_at_10_std": -0.230798, + "nauc_map_at_10_diff1": 0.83612, + "nauc_map_at_20_max": 0.622049, + "nauc_map_at_20_std": -0.228016, + "nauc_map_at_20_diff1": 0.836072, + "nauc_map_at_100_max": 0.621177, + "nauc_map_at_100_std": -0.226716, + "nauc_map_at_100_diff1": 0.836405, + "nauc_map_at_1000_max": 0.621008, + "nauc_map_at_1000_std": -0.227002, + "nauc_map_at_1000_diff1": 0.836403, + "nauc_recall_at_1_max": 0.554284, + "nauc_recall_at_1_std": -0.251232, + "nauc_recall_at_1_diff1": 0.853441, + "nauc_recall_at_3_max": 0.687155, + "nauc_recall_at_3_std": -0.225337, + "nauc_recall_at_3_diff1": 0.807497, + "nauc_recall_at_5_max": 0.725775, + "nauc_recall_at_5_std": -0.208069, + "nauc_recall_at_5_diff1": 0.792642, + "nauc_recall_at_10_max": 0.766982, + "nauc_recall_at_10_std": -0.140617, + "nauc_recall_at_10_diff1": 0.790801, + "nauc_recall_at_20_max": 0.818312, + "nauc_recall_at_20_std": -0.040307, + "nauc_recall_at_20_diff1": 0.784199, + "nauc_recall_at_100_max": 0.884474, + "nauc_recall_at_100_std": 0.300658, + "nauc_recall_at_100_diff1": 0.807922, + "nauc_recall_at_1000_max": 0.967325, + "nauc_recall_at_1000_std": 0.658622, + "nauc_recall_at_1000_diff1": 0.744405, + "nauc_precision_at_1_max": 0.577069, + "nauc_precision_at_1_std": -0.227244, + "nauc_precision_at_1_diff1": 0.851721, + "nauc_precision_at_3_max": 0.611156, + "nauc_precision_at_3_std": -0.107513, + "nauc_precision_at_3_diff1": 0.595802, + "nauc_precision_at_5_max": 0.571612, + "nauc_precision_at_5_std": -0.032888, + "nauc_precision_at_5_diff1": 0.473374, + "nauc_precision_at_10_max": 0.502029, + "nauc_precision_at_10_std": 0.072119, + "nauc_precision_at_10_diff1": 0.332965, + "nauc_precision_at_20_max": 0.445596, + "nauc_precision_at_20_std": 0.193328, + "nauc_precision_at_20_diff1": 0.201162, + "nauc_precision_at_100_max": 0.223226, + "nauc_precision_at_100_std": 0.409127, + "nauc_precision_at_100_diff1": -0.090569, + "nauc_precision_at_1000_max": -0.021917, + "nauc_precision_at_1000_std": 0.359989, + "nauc_precision_at_1000_diff1": -0.338305, + "nauc_mrr_at_1_max": 0.577069, + "nauc_mrr_at_1_std": -0.227244, + "nauc_mrr_at_1_diff1": 0.851721, + "nauc_mrr_at_3_max": 0.626207, + "nauc_mrr_at_3_std": -0.222305, + "nauc_mrr_at_3_diff1": 0.836677, + "nauc_mrr_at_5_max": 0.630294, + "nauc_mrr_at_5_std": -0.217863, + "nauc_mrr_at_5_diff1": 0.835279, + "nauc_mrr_at_10_max": 0.62918, + "nauc_mrr_at_10_std": -0.216564, + "nauc_mrr_at_10_diff1": 0.836449, + "nauc_mrr_at_20_max": 0.628656, + "nauc_mrr_at_20_std": -0.214933, + "nauc_mrr_at_20_diff1": 0.836753, + "nauc_mrr_at_100_max": 0.627733, + "nauc_mrr_at_100_std": -0.214223, + "nauc_mrr_at_100_diff1": 0.837073, + "nauc_mrr_at_1000_max": 0.627551, + "nauc_mrr_at_1000_std": -0.214487, + "nauc_mrr_at_1000_diff1": 0.837061, + "hit_rate_at_1": 0.76139, + "hit_rate_at_3": 0.86595, + "hit_rate_at_5": 0.89477, + "hit_rate_at_10": 0.92493, + "hit_rate_at_20": 0.94236, + "hit_rate_at_100": 0.97788, + "hit_rate_at_1000": 0.99732, + "main_score": 0.84091, + "hf_subset": "pl", + "languages": [ + "pol-Latn" + ] + }, + { + "ndcg_at_1": 0.65248, + "ndcg_at_3": 0.73717, + "ndcg_at_5": 0.7569, + "ndcg_at_10": 0.77502, + "ndcg_at_20": 0.78473, + "ndcg_at_100": 0.79423, + "ndcg_at_1000": 0.79651, + "map_at_1": 0.63452, + "map_at_3": 0.71317, + "map_at_5": 0.72482, + "map_at_10": 0.73308, + "map_at_20": 0.73587, + "map_at_100": 0.73723, + "map_at_1000": 0.73734, + "recall_at_1": 0.63452, + "recall_at_3": 0.79638, + "recall_at_5": 0.84287, + "recall_at_10": 0.8963, + "recall_at_20": 0.93365, + "recall_at_100": 0.98377, + "recall_at_1000": 1.0, + "accuracy": 0.63452, + "precision_at_1": 0.65248, + "precision_at_3": 0.27849, + "precision_at_5": 0.17749, + "precision_at_10": 0.09518, + "precision_at_20": 0.04967, + "precision_at_100": 0.01051, + "precision_at_1000": 0.00107, + "mrr_at_1": 0.652482, + "mrr_at_3": 0.723089, + "mrr_at_5": 0.733231, + "mrr_at_10": 0.739896, + "mrr_at_20": 0.742459, + "mrr_at_100": 0.743658, + "mrr_at_1000": 0.743742, + "nauc_ndcg_at_1_max": 0.282278, + "nauc_ndcg_at_1_std": -0.379417, + "nauc_ndcg_at_1_diff1": 0.801701, + "nauc_ndcg_at_3_max": 0.301718, + "nauc_ndcg_at_3_std": -0.427872, + "nauc_ndcg_at_3_diff1": 0.763284, + "nauc_ndcg_at_5_max": 0.301537, + "nauc_ndcg_at_5_std": -0.435642, + "nauc_ndcg_at_5_diff1": 0.768044, + "nauc_ndcg_at_10_max": 0.304077, + "nauc_ndcg_at_10_std": -0.433991, + "nauc_ndcg_at_10_diff1": 0.770124, + "nauc_ndcg_at_20_max": 0.302833, + "nauc_ndcg_at_20_std": -0.430421, + "nauc_ndcg_at_20_diff1": 0.773769, + "nauc_ndcg_at_100_max": 0.299453, + "nauc_ndcg_at_100_std": -0.421852, + "nauc_ndcg_at_100_diff1": 0.775028, + "nauc_ndcg_at_1000_max": 0.298303, + "nauc_ndcg_at_1000_std": -0.420997, + "nauc_ndcg_at_1000_diff1": 0.775649, + "nauc_map_at_1_max": 0.250056, + "nauc_map_at_1_std": -0.368929, + "nauc_map_at_1_diff1": 0.801137, + "nauc_map_at_3_max": 0.288658, + "nauc_map_at_3_std": -0.416728, + "nauc_map_at_3_diff1": 0.774794, + "nauc_map_at_5_max": 0.290083, + "nauc_map_at_5_std": -0.419596, + "nauc_map_at_5_diff1": 0.777698, + "nauc_map_at_10_max": 0.291275, + "nauc_map_at_10_std": -0.419127, + "nauc_map_at_10_diff1": 0.778421, + "nauc_map_at_20_max": 0.290797, + "nauc_map_at_20_std": -0.41822, + "nauc_map_at_20_diff1": 0.779425, + "nauc_map_at_100_max": 0.290348, + "nauc_map_at_100_std": -0.417295, + "nauc_map_at_100_diff1": 0.77957, + "nauc_map_at_1000_max": 0.29029, + "nauc_map_at_1000_std": -0.417241, + "nauc_map_at_1000_diff1": 0.779601, + "nauc_recall_at_1_max": 0.250056, + "nauc_recall_at_1_std": -0.368929, + "nauc_recall_at_1_diff1": 0.801137, + "nauc_recall_at_3_max": 0.326058, + "nauc_recall_at_3_std": -0.46849, + "nauc_recall_at_3_diff1": 0.719796, + "nauc_recall_at_5_max": 0.328291, + "nauc_recall_at_5_std": -0.504999, + "nauc_recall_at_5_diff1": 0.723301, + "nauc_recall_at_10_max": 0.354368, + "nauc_recall_at_10_std": -0.533825, + "nauc_recall_at_10_diff1": 0.723713, + "nauc_recall_at_20_max": 0.370343, + "nauc_recall_at_20_std": -0.550592, + "nauc_recall_at_20_diff1": 0.738482, + "nauc_recall_at_100_max": 0.386852, + "nauc_recall_at_100_std": -0.458011, + "nauc_recall_at_100_diff1": 0.719325, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.282278, + "nauc_precision_at_1_std": -0.379417, + "nauc_precision_at_1_diff1": 0.801701, + "nauc_precision_at_3_max": 0.332446, + "nauc_precision_at_3_std": -0.386066, + "nauc_precision_at_3_diff1": 0.519046, + "nauc_precision_at_5_max": 0.313937, + "nauc_precision_at_5_std": -0.363343, + "nauc_precision_at_5_diff1": 0.430126, + "nauc_precision_at_10_max": 0.272566, + "nauc_precision_at_10_std": -0.260899, + "nauc_precision_at_10_diff1": 0.246581, + "nauc_precision_at_20_max": 0.237394, + "nauc_precision_at_20_std": -0.171312, + "nauc_precision_at_20_diff1": 0.109069, + "nauc_precision_at_100_max": 0.1417, + "nauc_precision_at_100_std": 0.095097, + "nauc_precision_at_100_diff1": -0.20532, + "nauc_precision_at_1000_max": 0.084177, + "nauc_precision_at_1000_std": 0.191302, + "nauc_precision_at_1000_diff1": -0.331213, + "nauc_mrr_at_1_max": 0.282278, + "nauc_mrr_at_1_std": -0.379417, + "nauc_mrr_at_1_diff1": 0.801701, + "nauc_mrr_at_3_max": 0.306452, + "nauc_mrr_at_3_std": -0.413356, + "nauc_mrr_at_3_diff1": 0.772509, + "nauc_mrr_at_5_max": 0.303326, + "nauc_mrr_at_5_std": -0.41726, + "nauc_mrr_at_5_diff1": 0.774784, + "nauc_mrr_at_10_max": 0.303066, + "nauc_mrr_at_10_std": -0.414874, + "nauc_mrr_at_10_diff1": 0.776307, + "nauc_mrr_at_20_max": 0.302647, + "nauc_mrr_at_20_std": -0.413866, + "nauc_mrr_at_20_diff1": 0.777188, + "nauc_mrr_at_100_max": 0.30204, + "nauc_mrr_at_100_std": -0.413343, + "nauc_mrr_at_100_diff1": 0.777432, + "nauc_mrr_at_1000_max": 0.302011, + "nauc_mrr_at_1000_std": -0.413348, + "nauc_mrr_at_1000_diff1": 0.777474, + "hit_rate_at_1": 0.65248, + "hit_rate_at_3": 0.80851, + "hit_rate_at_5": 0.85248, + "hit_rate_at_10": 0.90213, + "hit_rate_at_20": 0.93853, + "hit_rate_at_100": 0.98629, + "hit_rate_at_1000": 1.0, + "main_score": 0.77502, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ] + } + ] + }, + "evaluation_time": 210.00080180168152, + "kg_co2_emissions": null, + "date": 1777964765.039515 +} \ No newline at end of file diff --git a/results/BAAI__bge-m3/5617a9f61b028005a4858fdac845db406aefb181/model_meta.json b/results/BAAI__bge-m3/5617a9f61b028005a4858fdac845db406aefb181/model_meta.json index 8b8bb174de..08560b0d4a 100644 --- a/results/BAAI__bge-m3/5617a9f61b028005a4858fdac845db406aefb181/model_meta.json +++ b/results/BAAI__bge-m3/5617a9f61b028005a4858fdac845db406aefb181/model_meta.json @@ -1 +1,98 @@ -{"loader_kwargs": {}, "name": "BAAI/bge-m3", "revision": "5617a9f61b028005a4858fdac845db406aefb181", "release_date": "2024-06-28", "languages": ["afr-Latn", "amh-Ethi", "azj-Latn", "ast-Latn", "azj-Latn", "ben-Beng", "bul-Cyrl", "bel-Cyrl", "cat-Latn", "ceb-Latn", "ckb-Arab", "dan-Latn", "deu-Latn", "ell-Grek", "eng-Latn", "est-Latn", "fin-Latn", "fra-Latn", "glg-Latn", "guj-Gujr", "heb-Hebr", "hin-Deva", "ita-Latn", "jpn-Jpan", "kor-Hang", "rus-Cyrl", "tha-Thai", "ukr-Cyrl", "zho-Hans"], "n_parameters": 568000000, "memory_usage_mb": 2167.0, "max_tokens": 8194.0, "embed_dim": 1024, "license": "mit", "open_weights": true, "public_training_code": null, "public_training_data": "https://huggingface.co/datasets/cfli/bge-full-data", "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/BAAI/bge-m3", "similarity_fn_name": "cosine", "use_instructions": false, "training_datasets": ["NQ-NL", "CodeSearchNet", "T2Retrieval", "T2Reranking", "LeCaRDv2", "MIRACLRetrieval", "CMedQAv1-reranking", "NQHardNegatives", "NQ", "MSMARCOHardNegatives", "NQ-PL", "DuRetrieval", "MSMARCO", "CMedQAv2-reranking", "MIRACLRetrievalHardNegatives", "HotpotQA", "MrTidyRetrieval", "MIRACLReranking", "mMARCO-NL", "HotpotQA-PL", "MMarcoReranking", "NanoMSMARCORetrieval", "HotpotQAHardNegatives", "HotpotQA-NL", "NanoNQRetrieval", "MSMARCO-PL"], "adapted_from": null, "superseded_by": null, "modalities": ["text"], "is_cross_encoder": null, "citation": null, "contacts": null, "loader": "sentence_transformers_loader"} \ No newline at end of file +{ + "loader_kwargs": {}, + "name": "BAAI/bge-m3", + "revision": "5617a9f61b028005a4858fdac845db406aefb181", + "release_date": "2024-06-28", + "languages": [ + "afr-Latn", + "amh-Ethi", + "azj-Latn", + "ast-Latn", + "azj-Latn", + "ben-Beng", + "bul-Cyrl", + "bel-Cyrl", + "cat-Latn", + "ceb-Latn", + "ckb-Arab", + "dan-Latn", + "deu-Latn", + "ell-Grek", + "eng-Latn", + "est-Latn", + "fin-Latn", + "fra-Latn", + "glg-Latn", + "guj-Gujr", + "heb-Hebr", + "hin-Deva", + "ita-Latn", + "jpn-Jpan", + "kor-Hang", + "rus-Cyrl", + "tha-Thai", + "ukr-Cyrl", + "zho-Hans" + ], + "n_parameters": 568000000, + "n_active_parameters_override": null, + "n_embedding_parameters": 256002048, + "memory_usage_mb": 2167.0, + "max_tokens": 8194.0, + "embed_dim": 1024, + "license": "mit", + "open_weights": true, + "public_training_code": null, + "public_training_data": "https://huggingface.co/datasets/cfli/bge-full-data", + "framework": [ + "Sentence Transformers", + "PyTorch", + "ONNX" + ], + "reference": "https://huggingface.co/BAAI/bge-m3", + "similarity_fn_name": "cosine", + "use_instructions": false, + "training_datasets": [ + "MIRACLRetrievalHardNegatives", + "MIRACLRetrieval", + "NQ-PL", + "NQ", + "DuRetrieval", + "T2Retrieval", + "HotpotQA-PL", + "T2Reranking", + "CMedQAv1-reranking", + "MMarcoReranking", + "MSMARCO", + "NQHardNegatives", + "HotpotQA", + "MIRACLReranking", + "NQ-NL", + "HotpotQA-NL", + "NanoNQRetrieval", + "MSMARCO-PL", + "mMARCO-NL", + "CMedQAv2-reranking", + "HotpotQAHardNegatives", + "MrTidyRetrieval", + "MSMARCOHardNegatives", + "CodeSearchNet", + "LeCaRDv2", + "NanoMSMARCORetrieval" + ], + "adapted_from": null, + "superseded_by": null, + "modalities": [ + "text" + ], + "model_type": [ + "dense" + ], + "citation": "@misc{bge-m3,\n title={BGE M3-Embedding: Multi-Lingual, Multi-Functionality, Multi-Granularity Text Embeddings Through Self-Knowledge Distillation},\n author={Jianlv Chen and Shitao Xiao and Peitian Zhang and Kun Luo and Defu Lian and Zheng Liu},\n year={2024},\n eprint={2402.03216},\n archivePrefix={arXiv},\n primaryClass={cs.CL}\n}", + "contacts": null, + "experiment_kwargs": null, + "output_dtypes": null, + "extra_requirements_groups": null, + "loader": "SentenceTransformerEncoderWrapper", + "is_cross_encoder": false +} \ No newline at end of file diff --git a/results/Octen__Octen-Embedding-0.6B/1a00a4e837bd788f6f8d91bc43201a5e52cf8ef8/ElasticKBRetrieval.json b/results/Octen__Octen-Embedding-0.6B/1a00a4e837bd788f6f8d91bc43201a5e52cf8ef8/ElasticKBRetrieval.json new file mode 100644 index 0000000000..cbfdd63734 --- /dev/null +++ b/results/Octen__Octen-Embedding-0.6B/1a00a4e837bd788f6f8d91bc43201a5e52cf8ef8/ElasticKBRetrieval.json @@ -0,0 +1,324 @@ +{ + "dataset_revision": "21bdf1e024bf7c9f46720017559ce2f8c6116507", + "task_name": "ElasticKBRetrieval", + "mteb_version": "2.12.38", + "scores": { + "synthetic_test": [ + { + "ndcg_at_1": 0.73936, + "ndcg_at_3": 0.58816, + "ndcg_at_5": 0.55405, + "ndcg_at_10": 0.52799, + "ndcg_at_20": 0.52485, + "ndcg_at_100": 0.56973, + "ndcg_at_1000": 0.63374, + "map_at_1": 0.24816, + "map_at_3": 0.30388, + "map_at_5": 0.32181, + "map_at_10": 0.34222, + "map_at_20": 0.35969, + "map_at_100": 0.38758, + "map_at_1000": 0.39894, + "recall_at_1": 0.24816, + "recall_at_3": 0.32624, + "recall_at_5": 0.36605, + "recall_at_10": 0.41416, + "recall_at_20": 0.48326, + "recall_at_100": 0.65122, + "recall_at_1000": 0.84675, + "accuracy": 0.24816, + "precision_at_1": 0.73936, + "precision_at_3": 0.4344, + "precision_at_5": 0.33404, + "precision_at_10": 0.23351, + "precision_at_20": 0.16489, + "precision_at_100": 0.06356, + "precision_at_1000": 0.01218, + "mrr_at_1": 0.739362, + "mrr_at_3": 0.810284, + "mrr_at_5": 0.81773, + "mrr_at_10": 0.823698, + "mrr_at_20": 0.825783, + "mrr_at_100": 0.825916, + "mrr_at_1000": 0.825916, + "nauc_ndcg_at_1_max": 0.281403, + "nauc_ndcg_at_1_std": -0.219404, + "nauc_ndcg_at_1_diff1": 0.790233, + "nauc_ndcg_at_3_max": 0.209094, + "nauc_ndcg_at_3_std": -0.047725, + "nauc_ndcg_at_3_diff1": 0.356419, + "nauc_ndcg_at_5_max": 0.176717, + "nauc_ndcg_at_5_std": -0.043048, + "nauc_ndcg_at_5_diff1": 0.372745, + "nauc_ndcg_at_10_max": 0.20584, + "nauc_ndcg_at_10_std": -0.040865, + "nauc_ndcg_at_10_diff1": 0.402461, + "nauc_ndcg_at_20_max": 0.230074, + "nauc_ndcg_at_20_std": -0.045791, + "nauc_ndcg_at_20_diff1": 0.445532, + "nauc_ndcg_at_100_max": 0.289317, + "nauc_ndcg_at_100_std": 0.060374, + "nauc_ndcg_at_100_diff1": 0.448364, + "nauc_ndcg_at_1000_max": 0.237305, + "nauc_ndcg_at_1000_std": 0.095315, + "nauc_ndcg_at_1000_diff1": 0.403294, + "nauc_map_at_1_max": 0.148002, + "nauc_map_at_1_std": -0.182529, + "nauc_map_at_1_diff1": 0.55302, + "nauc_map_at_3_max": 0.204183, + "nauc_map_at_3_std": -0.149393, + "nauc_map_at_3_diff1": 0.500615, + "nauc_map_at_5_max": 0.203984, + "nauc_map_at_5_std": -0.1431, + "nauc_map_at_5_diff1": 0.501691, + "nauc_map_at_10_max": 0.217263, + "nauc_map_at_10_std": -0.115665, + "nauc_map_at_10_diff1": 0.479076, + "nauc_map_at_20_max": 0.230306, + "nauc_map_at_20_std": -0.088828, + "nauc_map_at_20_diff1": 0.469056, + "nauc_map_at_100_max": 0.243975, + "nauc_map_at_100_std": -0.028448, + "nauc_map_at_100_diff1": 0.450593, + "nauc_map_at_1000_max": 0.235089, + "nauc_map_at_1000_std": -0.018376, + "nauc_map_at_1000_diff1": 0.444871, + "nauc_recall_at_1_max": 0.148002, + "nauc_recall_at_1_std": -0.182529, + "nauc_recall_at_1_diff1": 0.55302, + "nauc_recall_at_3_max": 0.213949, + "nauc_recall_at_3_std": -0.124728, + "nauc_recall_at_3_diff1": 0.459844, + "nauc_recall_at_5_max": 0.202502, + "nauc_recall_at_5_std": -0.117526, + "nauc_recall_at_5_diff1": 0.44017, + "nauc_recall_at_10_max": 0.20767, + "nauc_recall_at_10_std": -0.07605, + "nauc_recall_at_10_diff1": 0.382588, + "nauc_recall_at_20_max": 0.238498, + "nauc_recall_at_20_std": -0.025144, + "nauc_recall_at_20_diff1": 0.33617, + "nauc_recall_at_100_max": 0.297485, + "nauc_recall_at_100_std": 0.170836, + "nauc_recall_at_100_diff1": 0.321857, + "nauc_recall_at_1000_max": 0.118082, + "nauc_recall_at_1000_std": 0.301483, + "nauc_recall_at_1000_diff1": 0.106929, + "nauc_precision_at_1_max": 0.281403, + "nauc_precision_at_1_std": -0.219404, + "nauc_precision_at_1_diff1": 0.790233, + "nauc_precision_at_3_max": 0.138343, + "nauc_precision_at_3_std": 0.082451, + "nauc_precision_at_3_diff1": -0.032465, + "nauc_precision_at_5_max": 0.035583, + "nauc_precision_at_5_std": 0.160659, + "nauc_precision_at_5_diff1": -0.148196, + "nauc_precision_at_10_max": 0.017511, + "nauc_precision_at_10_std": 0.25049, + "nauc_precision_at_10_diff1": -0.294507, + "nauc_precision_at_20_max": -0.041792, + "nauc_precision_at_20_std": 0.299093, + "nauc_precision_at_20_diff1": -0.348326, + "nauc_precision_at_100_max": -0.13837, + "nauc_precision_at_100_std": 0.313166, + "nauc_precision_at_100_diff1": -0.34537, + "nauc_precision_at_1000_max": -0.27537, + "nauc_precision_at_1000_std": 0.12354, + "nauc_precision_at_1000_diff1": -0.331223, + "nauc_mrr_at_1_max": 0.281403, + "nauc_mrr_at_1_std": -0.219404, + "nauc_mrr_at_1_diff1": 0.790233, + "nauc_mrr_at_3_max": 0.334812, + "nauc_mrr_at_3_std": -0.159168, + "nauc_mrr_at_3_diff1": 0.767054, + "nauc_mrr_at_5_max": 0.34137, + "nauc_mrr_at_5_std": -0.151331, + "nauc_mrr_at_5_diff1": 0.776198, + "nauc_mrr_at_10_max": 0.336915, + "nauc_mrr_at_10_std": -0.166356, + "nauc_mrr_at_10_diff1": 0.7796, + "nauc_mrr_at_20_max": 0.330988, + "nauc_mrr_at_20_std": -0.16854, + "nauc_mrr_at_20_diff1": 0.778126, + "nauc_mrr_at_100_max": 0.331162, + "nauc_mrr_at_100_std": -0.168206, + "nauc_mrr_at_100_diff1": 0.777939, + "nauc_mrr_at_1000_max": 0.331162, + "nauc_mrr_at_1000_std": -0.168206, + "nauc_mrr_at_1000_diff1": 0.777939, + "hit_rate_at_1": 0.73936, + "hit_rate_at_3": 0.89362, + "hit_rate_at_5": 0.92553, + "hit_rate_at_10": 0.96809, + "hit_rate_at_20": 0.99468, + "hit_rate_at_100": 1.0, + "hit_rate_at_1000": 1.0, + "main_score": 0.52799, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ], + "real_chat_test": [ + { + "ndcg_at_1": 0.75, + "ndcg_at_3": 0.67637, + "ndcg_at_5": 0.64916, + "ndcg_at_10": 0.59006, + "ndcg_at_20": 0.54134, + "ndcg_at_100": 0.49647, + "ndcg_at_1000": 0.57731, + "map_at_1": 0.06219, + "map_at_3": 0.10988, + "map_at_5": 0.13378, + "map_at_10": 0.15983, + "map_at_20": 0.18029, + "map_at_100": 0.22967, + "map_at_1000": 0.28278, + "recall_at_1": 0.06219, + "recall_at_3": 0.11746, + "recall_at_5": 0.15345, + "recall_at_10": 0.19987, + "recall_at_20": 0.24448, + "recall_at_100": 0.40837, + "recall_at_1000": 0.72265, + "accuracy": 0.06219, + "precision_at_1": 0.75, + "precision_at_3": 0.64224, + "precision_at_5": 0.5931, + "precision_at_10": 0.48405, + "precision_at_20": 0.3847, + "precision_at_100": 0.22784, + "precision_at_1000": 0.07884, + "mrr_at_1": 0.75, + "mrr_at_3": 0.818966, + "mrr_at_5": 0.829095, + "mrr_at_10": 0.832918, + "mrr_at_20": 0.833226, + "mrr_at_100": 0.833401, + "mrr_at_1000": 0.833451, + "nauc_ndcg_at_1_max": 0.373385, + "nauc_ndcg_at_1_std": 0.173111, + "nauc_ndcg_at_1_diff1": 0.447635, + "nauc_ndcg_at_3_max": 0.232353, + "nauc_ndcg_at_3_std": 0.143706, + "nauc_ndcg_at_3_diff1": 0.094803, + "nauc_ndcg_at_5_max": 0.188863, + "nauc_ndcg_at_5_std": 0.168271, + "nauc_ndcg_at_5_diff1": 0.060165, + "nauc_ndcg_at_10_max": 0.217753, + "nauc_ndcg_at_10_std": 0.217391, + "nauc_ndcg_at_10_diff1": 0.043041, + "nauc_ndcg_at_20_max": 0.2072, + "nauc_ndcg_at_20_std": 0.2223, + "nauc_ndcg_at_20_diff1": 0.032207, + "nauc_ndcg_at_100_max": 0.272128, + "nauc_ndcg_at_100_std": 0.223089, + "nauc_ndcg_at_100_diff1": 0.088372, + "nauc_ndcg_at_1000_max": 0.280359, + "nauc_ndcg_at_1000_std": 0.319018, + "nauc_ndcg_at_1000_diff1": 0.164739, + "nauc_map_at_1_max": 0.319379, + "nauc_map_at_1_std": -0.026333, + "nauc_map_at_1_diff1": 0.4588, + "nauc_map_at_3_max": 0.266346, + "nauc_map_at_3_std": -0.020167, + "nauc_map_at_3_diff1": 0.322706, + "nauc_map_at_5_max": 0.247713, + "nauc_map_at_5_std": -0.028105, + "nauc_map_at_5_diff1": 0.297942, + "nauc_map_at_10_max": 0.266896, + "nauc_map_at_10_std": -0.015175, + "nauc_map_at_10_diff1": 0.253846, + "nauc_map_at_20_max": 0.263599, + "nauc_map_at_20_std": -0.001049, + "nauc_map_at_20_diff1": 0.24149, + "nauc_map_at_100_max": 0.275434, + "nauc_map_at_100_std": 0.096613, + "nauc_map_at_100_diff1": 0.22823, + "nauc_map_at_1000_max": 0.273229, + "nauc_map_at_1000_std": 0.19301, + "nauc_map_at_1000_diff1": 0.175988, + "nauc_recall_at_1_max": 0.319379, + "nauc_recall_at_1_std": -0.026333, + "nauc_recall_at_1_diff1": 0.4588, + "nauc_recall_at_3_max": 0.243764, + "nauc_recall_at_3_std": -0.0337, + "nauc_recall_at_3_diff1": 0.291543, + "nauc_recall_at_5_max": 0.217828, + "nauc_recall_at_5_std": -0.047963, + "nauc_recall_at_5_diff1": 0.270728, + "nauc_recall_at_10_max": 0.236085, + "nauc_recall_at_10_std": -0.034598, + "nauc_recall_at_10_diff1": 0.207474, + "nauc_recall_at_20_max": 0.210427, + "nauc_recall_at_20_std": -0.024158, + "nauc_recall_at_20_diff1": 0.184987, + "nauc_recall_at_100_max": 0.220813, + "nauc_recall_at_100_std": 0.098846, + "nauc_recall_at_100_diff1": 0.181412, + "nauc_recall_at_1000_max": 0.189384, + "nauc_recall_at_1000_std": 0.241622, + "nauc_recall_at_1000_diff1": 0.162889, + "nauc_precision_at_1_max": 0.373385, + "nauc_precision_at_1_std": 0.173111, + "nauc_precision_at_1_diff1": 0.447635, + "nauc_precision_at_3_max": 0.156225, + "nauc_precision_at_3_std": 0.141533, + "nauc_precision_at_3_diff1": -0.046126, + "nauc_precision_at_5_max": 0.086194, + "nauc_precision_at_5_std": 0.182089, + "nauc_precision_at_5_diff1": -0.113671, + "nauc_precision_at_10_max": 0.080268, + "nauc_precision_at_10_std": 0.244054, + "nauc_precision_at_10_diff1": -0.188392, + "nauc_precision_at_20_max": -0.000499, + "nauc_precision_at_20_std": 0.260767, + "nauc_precision_at_20_diff1": -0.204294, + "nauc_precision_at_100_max": -0.046315, + "nauc_precision_at_100_std": 0.283161, + "nauc_precision_at_100_diff1": -0.244871, + "nauc_precision_at_1000_max": -0.102824, + "nauc_precision_at_1000_std": 0.105176, + "nauc_precision_at_1000_diff1": -0.266003, + "nauc_mrr_at_1_max": 0.373385, + "nauc_mrr_at_1_std": 0.173111, + "nauc_mrr_at_1_diff1": 0.447635, + "nauc_mrr_at_3_max": 0.343678, + "nauc_mrr_at_3_std": 0.172372, + "nauc_mrr_at_3_diff1": 0.395259, + "nauc_mrr_at_5_max": 0.354083, + "nauc_mrr_at_5_std": 0.181841, + "nauc_mrr_at_5_diff1": 0.408383, + "nauc_mrr_at_10_max": 0.354954, + "nauc_mrr_at_10_std": 0.177271, + "nauc_mrr_at_10_diff1": 0.410678, + "nauc_mrr_at_20_max": 0.353912, + "nauc_mrr_at_20_std": 0.177976, + "nauc_mrr_at_20_diff1": 0.410387, + "nauc_mrr_at_100_max": 0.354773, + "nauc_mrr_at_100_std": 0.177899, + "nauc_mrr_at_100_diff1": 0.41068, + "nauc_mrr_at_1000_max": 0.354604, + "nauc_mrr_at_1000_std": 0.177932, + "nauc_mrr_at_1000_diff1": 0.410716, + "hit_rate_at_1": 0.75, + "hit_rate_at_3": 0.90948, + "hit_rate_at_5": 0.95259, + "hit_rate_at_10": 0.97845, + "hit_rate_at_20": 0.98276, + "hit_rate_at_100": 0.99138, + "hit_rate_at_1000": 1.0, + "main_score": 0.59006, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 359.3623237609863, + "kg_co2_emissions": null, + "date": 1777885201.832418 +} \ No newline at end of file diff --git a/results/Octen__Octen-Embedding-0.6B/1a00a4e837bd788f6f8d91bc43201a5e52cf8ef8/GlobalNewsRetrieval.json b/results/Octen__Octen-Embedding-0.6B/1a00a4e837bd788f6f8d91bc43201a5e52cf8ef8/GlobalNewsRetrieval.json new file mode 100644 index 0000000000..3465c7a361 --- /dev/null +++ b/results/Octen__Octen-Embedding-0.6B/1a00a4e837bd788f6f8d91bc43201a5e52cf8ef8/GlobalNewsRetrieval.json @@ -0,0 +1,1562 @@ +{ + "dataset_revision": "234dcd32922d32c78e08bfe506a2b90a5401cf7c", + "task_name": "GlobalNewsRetrieval", + "mteb_version": "2.12.38", + "scores": { + "test": [ + { + "ndcg_at_1": 0.64662, + "ndcg_at_3": 0.72773, + "ndcg_at_5": 0.74979, + "ndcg_at_10": 0.77165, + "ndcg_at_20": 0.78372, + "ndcg_at_100": 0.79296, + "ndcg_at_1000": 0.79389, + "map_at_1": 0.61487, + "map_at_3": 0.69834, + "map_at_5": 0.71255, + "map_at_10": 0.7231, + "map_at_20": 0.72659, + "map_at_100": 0.72812, + "map_at_1000": 0.72819, + "recall_at_1": 0.61487, + "recall_at_3": 0.78926, + "recall_at_5": 0.83855, + "recall_at_10": 0.90142, + "recall_at_20": 0.94758, + "recall_at_100": 0.99394, + "recall_at_1000": 1.0, + "accuracy": 0.61487, + "precision_at_1": 0.64662, + "precision_at_3": 0.2853, + "precision_at_5": 0.18471, + "precision_at_10": 0.10063, + "precision_at_20": 0.05301, + "precision_at_100": 0.01123, + "precision_at_1000": 0.00113, + "mrr_at_1": 0.646617, + "mrr_at_3": 0.722013, + "mrr_at_5": 0.732226, + "mrr_at_10": 0.739445, + "mrr_at_20": 0.742519, + "mrr_at_100": 0.743492, + "mrr_at_1000": 0.743517, + "nauc_ndcg_at_1_max": 0.513771, + "nauc_ndcg_at_1_std": -0.256062, + "nauc_ndcg_at_1_diff1": 0.771008, + "nauc_ndcg_at_3_max": 0.48717, + "nauc_ndcg_at_3_std": -0.273528, + "nauc_ndcg_at_3_diff1": 0.715482, + "nauc_ndcg_at_5_max": 0.506105, + "nauc_ndcg_at_5_std": -0.258033, + "nauc_ndcg_at_5_diff1": 0.708829, + "nauc_ndcg_at_10_max": 0.515622, + "nauc_ndcg_at_10_std": -0.258214, + "nauc_ndcg_at_10_diff1": 0.707031, + "nauc_ndcg_at_20_max": 0.517406, + "nauc_ndcg_at_20_std": -0.237813, + "nauc_ndcg_at_20_diff1": 0.710925, + "nauc_ndcg_at_100_max": 0.513661, + "nauc_ndcg_at_100_std": -0.247286, + "nauc_ndcg_at_100_diff1": 0.721611, + "nauc_ndcg_at_1000_max": 0.51191, + "nauc_ndcg_at_1000_std": -0.248297, + "nauc_ndcg_at_1000_diff1": 0.722582, + "nauc_map_at_1_max": 0.426214, + "nauc_map_at_1_std": -0.247957, + "nauc_map_at_1_diff1": 0.756314, + "nauc_map_at_3_max": 0.465534, + "nauc_map_at_3_std": -0.26973, + "nauc_map_at_3_diff1": 0.726855, + "nauc_map_at_5_max": 0.480926, + "nauc_map_at_5_std": -0.262445, + "nauc_map_at_5_diff1": 0.724678, + "nauc_map_at_10_max": 0.487767, + "nauc_map_at_10_std": -0.261641, + "nauc_map_at_10_diff1": 0.724445, + "nauc_map_at_20_max": 0.488522, + "nauc_map_at_20_std": -0.256323, + "nauc_map_at_20_diff1": 0.7257, + "nauc_map_at_100_max": 0.488199, + "nauc_map_at_100_std": -0.257278, + "nauc_map_at_100_diff1": 0.727221, + "nauc_map_at_1000_max": 0.488115, + "nauc_map_at_1000_std": -0.257252, + "nauc_map_at_1000_diff1": 0.727255, + "nauc_recall_at_1_max": 0.426214, + "nauc_recall_at_1_std": -0.247957, + "nauc_recall_at_1_diff1": 0.756314, + "nauc_recall_at_3_max": 0.477761, + "nauc_recall_at_3_std": -0.290944, + "nauc_recall_at_3_diff1": 0.653862, + "nauc_recall_at_5_max": 0.519941, + "nauc_recall_at_5_std": -0.249313, + "nauc_recall_at_5_diff1": 0.611815, + "nauc_recall_at_10_max": 0.573135, + "nauc_recall_at_10_std": -0.255768, + "nauc_recall_at_10_diff1": 0.562178, + "nauc_recall_at_20_max": 0.627573, + "nauc_recall_at_20_std": -0.003115, + "nauc_recall_at_20_diff1": 0.487708, + "nauc_recall_at_100_max": 0.856938, + "nauc_recall_at_100_std": 0.0545, + "nauc_recall_at_100_diff1": 0.506939, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.513771, + "nauc_precision_at_1_std": -0.256062, + "nauc_precision_at_1_diff1": 0.771008, + "nauc_precision_at_3_max": 0.502583, + "nauc_precision_at_3_std": -0.211147, + "nauc_precision_at_3_diff1": 0.436936, + "nauc_precision_at_5_max": 0.464042, + "nauc_precision_at_5_std": -0.101861, + "nauc_precision_at_5_diff1": 0.256337, + "nauc_precision_at_10_max": 0.365811, + "nauc_precision_at_10_std": -0.021238, + "nauc_precision_at_10_diff1": 0.048467, + "nauc_precision_at_20_max": 0.30602, + "nauc_precision_at_20_std": 0.125328, + "nauc_precision_at_20_diff1": -0.100427, + "nauc_precision_at_100_max": 0.154206, + "nauc_precision_at_100_std": 0.147582, + "nauc_precision_at_100_diff1": -0.240089, + "nauc_precision_at_1000_max": 0.12166, + "nauc_precision_at_1000_std": 0.156542, + "nauc_precision_at_1000_diff1": -0.259291, + "nauc_mrr_at_1_max": 0.513771, + "nauc_mrr_at_1_std": -0.256062, + "nauc_mrr_at_1_diff1": 0.771008, + "nauc_mrr_at_3_max": 0.536439, + "nauc_mrr_at_3_std": -0.263329, + "nauc_mrr_at_3_diff1": 0.735594, + "nauc_mrr_at_5_max": 0.539503, + "nauc_mrr_at_5_std": -0.252145, + "nauc_mrr_at_5_diff1": 0.731378, + "nauc_mrr_at_10_max": 0.53837, + "nauc_mrr_at_10_std": -0.253879, + "nauc_mrr_at_10_diff1": 0.7308, + "nauc_mrr_at_20_max": 0.538104, + "nauc_mrr_at_20_std": -0.249273, + "nauc_mrr_at_20_diff1": 0.731955, + "nauc_mrr_at_100_max": 0.537836, + "nauc_mrr_at_100_std": -0.250377, + "nauc_mrr_at_100_diff1": 0.733473, + "nauc_mrr_at_1000_max": 0.537782, + "nauc_mrr_at_1000_std": -0.250491, + "nauc_mrr_at_1000_diff1": 0.733523, + "hit_rate_at_1": 0.64662, + "hit_rate_at_3": 0.8183, + "hit_rate_at_5": 0.86216, + "hit_rate_at_10": 0.91479, + "hit_rate_at_20": 0.95865, + "hit_rate_at_100": 0.99624, + "hit_rate_at_1000": 1.0, + "main_score": 0.77165, + "hf_subset": "ar", + "languages": [ + "ara-Arab" + ] + }, + { + "ndcg_at_1": 0.76452, + "ndcg_at_3": 0.81804, + "ndcg_at_5": 0.83398, + "ndcg_at_10": 0.84388, + "ndcg_at_20": 0.8528, + "ndcg_at_100": 0.86063, + "ndcg_at_1000": 0.8622, + "map_at_1": 0.70887, + "map_at_3": 0.79573, + "map_at_5": 0.80675, + "map_at_10": 0.81176, + "map_at_20": 0.81464, + "map_at_100": 0.81596, + "map_at_1000": 0.81606, + "recall_at_1": 0.70887, + "recall_at_3": 0.85726, + "recall_at_5": 0.89274, + "recall_at_10": 0.91998, + "recall_at_20": 0.95108, + "recall_at_100": 0.98978, + "recall_at_1000": 1.0, + "accuracy": 0.70887, + "precision_at_1": 0.76452, + "precision_at_3": 0.32401, + "precision_at_5": 0.20473, + "precision_at_10": 0.10645, + "precision_at_20": 0.0557, + "precision_at_100": 0.01168, + "precision_at_1000": 0.00118, + "mrr_at_1": 0.764516, + "mrr_at_3": 0.820609, + "mrr_at_5": 0.826523, + "mrr_at_10": 0.829957, + "mrr_at_20": 0.832098, + "mrr_at_100": 0.832829, + "mrr_at_1000": 0.832878, + "nauc_ndcg_at_1_max": 0.525094, + "nauc_ndcg_at_1_std": -0.599205, + "nauc_ndcg_at_1_diff1": 0.840474, + "nauc_ndcg_at_3_max": 0.502742, + "nauc_ndcg_at_3_std": -0.677327, + "nauc_ndcg_at_3_diff1": 0.792403, + "nauc_ndcg_at_5_max": 0.54395, + "nauc_ndcg_at_5_std": -0.680045, + "nauc_ndcg_at_5_diff1": 0.800699, + "nauc_ndcg_at_10_max": 0.54456, + "nauc_ndcg_at_10_std": -0.658539, + "nauc_ndcg_at_10_diff1": 0.799204, + "nauc_ndcg_at_20_max": 0.532133, + "nauc_ndcg_at_20_std": -0.658054, + "nauc_ndcg_at_20_diff1": 0.800957, + "nauc_ndcg_at_100_max": 0.526049, + "nauc_ndcg_at_100_std": -0.655201, + "nauc_ndcg_at_100_diff1": 0.803034, + "nauc_ndcg_at_1000_max": 0.526659, + "nauc_ndcg_at_1000_std": -0.65008, + "nauc_ndcg_at_1000_diff1": 0.80395, + "nauc_map_at_1_max": 0.399091, + "nauc_map_at_1_std": -0.526621, + "nauc_map_at_1_diff1": 0.814761, + "nauc_map_at_3_max": 0.487674, + "nauc_map_at_3_std": -0.644144, + "nauc_map_at_3_diff1": 0.797605, + "nauc_map_at_5_max": 0.514534, + "nauc_map_at_5_std": -0.645271, + "nauc_map_at_5_diff1": 0.802618, + "nauc_map_at_10_max": 0.515615, + "nauc_map_at_10_std": -0.63584, + "nauc_map_at_10_diff1": 0.80138, + "nauc_map_at_20_max": 0.51197, + "nauc_map_at_20_std": -0.635565, + "nauc_map_at_20_diff1": 0.801844, + "nauc_map_at_100_max": 0.510973, + "nauc_map_at_100_std": -0.635089, + "nauc_map_at_100_diff1": 0.80206, + "nauc_map_at_1000_max": 0.511014, + "nauc_map_at_1000_std": -0.634819, + "nauc_map_at_1000_diff1": 0.802096, + "nauc_recall_at_1_max": 0.399091, + "nauc_recall_at_1_std": -0.526621, + "nauc_recall_at_1_diff1": 0.814761, + "nauc_recall_at_3_max": 0.491623, + "nauc_recall_at_3_std": -0.762277, + "nauc_recall_at_3_diff1": 0.744396, + "nauc_recall_at_5_max": 0.617013, + "nauc_recall_at_5_std": -0.793186, + "nauc_recall_at_5_diff1": 0.757793, + "nauc_recall_at_10_max": 0.638697, + "nauc_recall_at_10_std": -0.716324, + "nauc_recall_at_10_diff1": 0.739739, + "nauc_recall_at_20_max": 0.58875, + "nauc_recall_at_20_std": -0.763572, + "nauc_recall_at_20_diff1": 0.737044, + "nauc_recall_at_100_max": 0.487985, + "nauc_recall_at_100_std": -1.061944, + "nauc_recall_at_100_diff1": 0.709175, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.525094, + "nauc_precision_at_1_std": -0.599205, + "nauc_precision_at_1_diff1": 0.840474, + "nauc_precision_at_3_max": 0.343823, + "nauc_precision_at_3_std": -0.333399, + "nauc_precision_at_3_diff1": 0.163185, + "nauc_precision_at_5_max": 0.318069, + "nauc_precision_at_5_std": -0.216935, + "nauc_precision_at_5_diff1": 0.040847, + "nauc_precision_at_10_max": 0.255111, + "nauc_precision_at_10_std": -0.088037, + "nauc_precision_at_10_diff1": -0.068929, + "nauc_precision_at_20_max": 0.13554, + "nauc_precision_at_20_std": 0.022048, + "nauc_precision_at_20_diff1": -0.199108, + "nauc_precision_at_100_max": 0.019049, + "nauc_precision_at_100_std": 0.14988, + "nauc_precision_at_100_diff1": -0.335953, + "nauc_precision_at_1000_max": -0.001636, + "nauc_precision_at_1000_std": 0.199193, + "nauc_precision_at_1000_diff1": -0.364354, + "nauc_mrr_at_1_max": 0.525094, + "nauc_mrr_at_1_std": -0.599205, + "nauc_mrr_at_1_diff1": 0.840474, + "nauc_mrr_at_3_max": 0.5347, + "nauc_mrr_at_3_std": -0.671215, + "nauc_mrr_at_3_diff1": 0.814177, + "nauc_mrr_at_5_max": 0.548026, + "nauc_mrr_at_5_std": -0.669069, + "nauc_mrr_at_5_diff1": 0.817366, + "nauc_mrr_at_10_max": 0.54574, + "nauc_mrr_at_10_std": -0.662047, + "nauc_mrr_at_10_diff1": 0.817789, + "nauc_mrr_at_20_max": 0.543233, + "nauc_mrr_at_20_std": -0.661687, + "nauc_mrr_at_20_diff1": 0.818393, + "nauc_mrr_at_100_max": 0.542679, + "nauc_mrr_at_100_std": -0.661179, + "nauc_mrr_at_100_diff1": 0.81853, + "nauc_mrr_at_1000_max": 0.542648, + "nauc_mrr_at_1000_std": -0.661009, + "nauc_mrr_at_1000_diff1": 0.818551, + "hit_rate_at_1": 0.76452, + "hit_rate_at_3": 0.88495, + "hit_rate_at_5": 0.91075, + "hit_rate_at_10": 0.93548, + "hit_rate_at_20": 0.96559, + "hit_rate_at_100": 0.99355, + "hit_rate_at_1000": 1.0, + "main_score": 0.84388, + "hf_subset": "zh", + "languages": [ + "zho-Hans" + ] + }, + { + "ndcg_at_1": 0.70709, + "ndcg_at_3": 0.77759, + "ndcg_at_5": 0.79404, + "ndcg_at_10": 0.81357, + "ndcg_at_20": 0.8236, + "ndcg_at_100": 0.831, + "ndcg_at_1000": 0.83159, + "map_at_1": 0.68097, + "map_at_3": 0.75462, + "map_at_5": 0.76519, + "map_at_10": 0.77405, + "map_at_20": 0.77706, + "map_at_100": 0.77837, + "map_at_1000": 0.77841, + "recall_at_1": 0.68097, + "recall_at_3": 0.82828, + "recall_at_5": 0.86554, + "recall_at_10": 0.92155, + "recall_at_20": 0.95975, + "recall_at_100": 0.99589, + "recall_at_1000": 1.0, + "accuracy": 0.68097, + "precision_at_1": 0.70709, + "precision_at_3": 0.29291, + "precision_at_5": 0.18561, + "precision_at_10": 0.09969, + "precision_at_20": 0.05216, + "precision_at_100": 0.01088, + "precision_at_1000": 0.00109, + "mrr_at_1": 0.707091, + "mrr_at_3": 0.770641, + "mrr_at_5": 0.778092, + "mrr_at_10": 0.786149, + "mrr_at_20": 0.788454, + "mrr_at_100": 0.789349, + "mrr_at_1000": 0.789375, + "nauc_ndcg_at_1_max": 0.438158, + "nauc_ndcg_at_1_std": -0.415162, + "nauc_ndcg_at_1_diff1": 0.813844, + "nauc_ndcg_at_3_max": 0.479797, + "nauc_ndcg_at_3_std": -0.465049, + "nauc_ndcg_at_3_diff1": 0.777808, + "nauc_ndcg_at_5_max": 0.502021, + "nauc_ndcg_at_5_std": -0.457564, + "nauc_ndcg_at_5_diff1": 0.77869, + "nauc_ndcg_at_10_max": 0.479219, + "nauc_ndcg_at_10_std": -0.467422, + "nauc_ndcg_at_10_diff1": 0.779994, + "nauc_ndcg_at_20_max": 0.476434, + "nauc_ndcg_at_20_std": -0.455225, + "nauc_ndcg_at_20_diff1": 0.782745, + "nauc_ndcg_at_100_max": 0.477918, + "nauc_ndcg_at_100_std": -0.445561, + "nauc_ndcg_at_100_diff1": 0.785826, + "nauc_ndcg_at_1000_max": 0.476579, + "nauc_ndcg_at_1000_std": -0.445852, + "nauc_ndcg_at_1000_diff1": 0.785563, + "nauc_map_at_1_max": 0.399425, + "nauc_map_at_1_std": -0.410482, + "nauc_map_at_1_diff1": 0.807254, + "nauc_map_at_3_max": 0.459814, + "nauc_map_at_3_std": -0.454494, + "nauc_map_at_3_diff1": 0.78308, + "nauc_map_at_5_max": 0.474581, + "nauc_map_at_5_std": -0.448769, + "nauc_map_at_5_diff1": 0.784257, + "nauc_map_at_10_max": 0.465005, + "nauc_map_at_10_std": -0.452066, + "nauc_map_at_10_diff1": 0.784472, + "nauc_map_at_20_max": 0.464754, + "nauc_map_at_20_std": -0.448608, + "nauc_map_at_20_diff1": 0.785067, + "nauc_map_at_100_max": 0.465164, + "nauc_map_at_100_std": -0.447287, + "nauc_map_at_100_diff1": 0.785689, + "nauc_map_at_1000_max": 0.465107, + "nauc_map_at_1000_std": -0.447286, + "nauc_map_at_1000_diff1": 0.785674, + "nauc_recall_at_1_max": 0.399425, + "nauc_recall_at_1_std": -0.410482, + "nauc_recall_at_1_diff1": 0.807254, + "nauc_recall_at_3_max": 0.512261, + "nauc_recall_at_3_std": -0.519962, + "nauc_recall_at_3_diff1": 0.736682, + "nauc_recall_at_5_max": 0.590948, + "nauc_recall_at_5_std": -0.518645, + "nauc_recall_at_5_diff1": 0.73822, + "nauc_recall_at_10_max": 0.507435, + "nauc_recall_at_10_std": -0.620507, + "nauc_recall_at_10_diff1": 0.72193, + "nauc_recall_at_20_max": 0.492954, + "nauc_recall_at_20_std": -0.603614, + "nauc_recall_at_20_diff1": 0.725608, + "nauc_recall_at_100_max": 0.819073, + "nauc_recall_at_100_std": -0.333047, + "nauc_recall_at_100_diff1": 0.846819, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.438158, + "nauc_precision_at_1_std": -0.415162, + "nauc_precision_at_1_diff1": 0.813844, + "nauc_precision_at_3_max": 0.459821, + "nauc_precision_at_3_std": -0.348827, + "nauc_precision_at_3_diff1": 0.467745, + "nauc_precision_at_5_max": 0.442887, + "nauc_precision_at_5_std": -0.218369, + "nauc_precision_at_5_diff1": 0.317796, + "nauc_precision_at_10_max": 0.269078, + "nauc_precision_at_10_std": -0.119037, + "nauc_precision_at_10_diff1": 0.10157, + "nauc_precision_at_20_max": 0.154562, + "nauc_precision_at_20_std": 0.067615, + "nauc_precision_at_20_diff1": -0.094549, + "nauc_precision_at_100_max": 0.036702, + "nauc_precision_at_100_std": 0.257061, + "nauc_precision_at_100_diff1": -0.285529, + "nauc_precision_at_1000_max": 0.011343, + "nauc_precision_at_1000_std": 0.271966, + "nauc_precision_at_1000_diff1": -0.311076, + "nauc_mrr_at_1_max": 0.438158, + "nauc_mrr_at_1_std": -0.415162, + "nauc_mrr_at_1_diff1": 0.813844, + "nauc_mrr_at_3_max": 0.486, + "nauc_mrr_at_3_std": -0.437988, + "nauc_mrr_at_3_diff1": 0.790002, + "nauc_mrr_at_5_max": 0.491852, + "nauc_mrr_at_5_std": -0.435371, + "nauc_mrr_at_5_diff1": 0.790034, + "nauc_mrr_at_10_max": 0.483177, + "nauc_mrr_at_10_std": -0.437761, + "nauc_mrr_at_10_diff1": 0.791514, + "nauc_mrr_at_20_max": 0.481306, + "nauc_mrr_at_20_std": -0.435584, + "nauc_mrr_at_20_diff1": 0.792108, + "nauc_mrr_at_100_max": 0.481548, + "nauc_mrr_at_100_std": -0.434808, + "nauc_mrr_at_100_diff1": 0.792336, + "nauc_mrr_at_1000_max": 0.481502, + "nauc_mrr_at_1000_std": -0.434843, + "nauc_mrr_at_1000_diff1": 0.792323, + "hit_rate_at_1": 0.70709, + "hit_rate_at_3": 0.84687, + "hit_rate_at_5": 0.87873, + "hit_rate_at_10": 0.93422, + "hit_rate_at_20": 0.96814, + "hit_rate_at_100": 0.99692, + "hit_rate_at_1000": 1.0, + "main_score": 0.81357, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.71093, + "ndcg_at_3": 0.79372, + "ndcg_at_5": 0.81337, + "ndcg_at_10": 0.82735, + "ndcg_at_20": 0.83485, + "ndcg_at_100": 0.84072, + "ndcg_at_1000": 0.84137, + "map_at_1": 0.69033, + "map_at_3": 0.7691, + "map_at_5": 0.78136, + "map_at_10": 0.78807, + "map_at_20": 0.79038, + "map_at_100": 0.79137, + "map_at_1000": 0.7914, + "recall_at_1": 0.69033, + "recall_at_3": 0.85257, + "recall_at_5": 0.89768, + "recall_at_10": 0.9382, + "recall_at_20": 0.96629, + "recall_at_100": 0.9954, + "recall_at_1000": 1.0, + "accuracy": 0.69033, + "precision_at_1": 0.71093, + "precision_at_3": 0.29894, + "precision_at_5": 0.19019, + "precision_at_10": 0.1001, + "precision_at_20": 0.05169, + "precision_at_100": 0.01067, + "precision_at_1000": 0.00107, + "mrr_at_1": 0.71093, + "mrr_at_3": 0.78141, + "mrr_at_5": 0.790705, + "mrr_at_10": 0.795585, + "mrr_at_20": 0.79746, + "mrr_at_100": 0.798276, + "mrr_at_1000": 0.798305, + "nauc_ndcg_at_1_max": 0.468811, + "nauc_ndcg_at_1_std": -0.283114, + "nauc_ndcg_at_1_diff1": 0.756843, + "nauc_ndcg_at_3_max": 0.500368, + "nauc_ndcg_at_3_std": -0.359806, + "nauc_ndcg_at_3_diff1": 0.729632, + "nauc_ndcg_at_5_max": 0.506318, + "nauc_ndcg_at_5_std": -0.343602, + "nauc_ndcg_at_5_diff1": 0.72956, + "nauc_ndcg_at_10_max": 0.51807, + "nauc_ndcg_at_10_std": -0.330027, + "nauc_ndcg_at_10_diff1": 0.736355, + "nauc_ndcg_at_20_max": 0.512228, + "nauc_ndcg_at_20_std": -0.32646, + "nauc_ndcg_at_20_diff1": 0.735308, + "nauc_ndcg_at_100_max": 0.504033, + "nauc_ndcg_at_100_std": -0.321762, + "nauc_ndcg_at_100_diff1": 0.737853, + "nauc_ndcg_at_1000_max": 0.502848, + "nauc_ndcg_at_1000_std": -0.32309, + "nauc_ndcg_at_1000_diff1": 0.737926, + "nauc_map_at_1_max": 0.430773, + "nauc_map_at_1_std": -0.302682, + "nauc_map_at_1_diff1": 0.747473, + "nauc_map_at_3_max": 0.486515, + "nauc_map_at_3_std": -0.345884, + "nauc_map_at_3_diff1": 0.732646, + "nauc_map_at_5_max": 0.489135, + "nauc_map_at_5_std": -0.335323, + "nauc_map_at_5_diff1": 0.732357, + "nauc_map_at_10_max": 0.494317, + "nauc_map_at_10_std": -0.329281, + "nauc_map_at_10_diff1": 0.735237, + "nauc_map_at_20_max": 0.49309, + "nauc_map_at_20_std": -0.327684, + "nauc_map_at_20_diff1": 0.734911, + "nauc_map_at_100_max": 0.492174, + "nauc_map_at_100_std": -0.327161, + "nauc_map_at_100_diff1": 0.735405, + "nauc_map_at_1000_max": 0.492128, + "nauc_map_at_1000_std": -0.327205, + "nauc_map_at_1000_diff1": 0.735414, + "nauc_recall_at_1_max": 0.430773, + "nauc_recall_at_1_std": -0.302682, + "nauc_recall_at_1_diff1": 0.747473, + "nauc_recall_at_3_max": 0.535103, + "nauc_recall_at_3_std": -0.444103, + "nauc_recall_at_3_diff1": 0.697067, + "nauc_recall_at_5_max": 0.570273, + "nauc_recall_at_5_std": -0.423196, + "nauc_recall_at_5_diff1": 0.689345, + "nauc_recall_at_10_max": 0.693842, + "nauc_recall_at_10_std": -0.378758, + "nauc_recall_at_10_diff1": 0.717743, + "nauc_recall_at_20_max": 0.729178, + "nauc_recall_at_20_std": -0.379631, + "nauc_recall_at_20_diff1": 0.686974, + "nauc_recall_at_100_max": 0.767802, + "nauc_recall_at_100_std": 0.019797, + "nauc_recall_at_100_diff1": 0.716948, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.468811, + "nauc_precision_at_1_std": -0.283114, + "nauc_precision_at_1_diff1": 0.756843, + "nauc_precision_at_3_max": 0.468484, + "nauc_precision_at_3_std": -0.24831, + "nauc_precision_at_3_diff1": 0.448227, + "nauc_precision_at_5_max": 0.385095, + "nauc_precision_at_5_std": -0.10905, + "nauc_precision_at_5_diff1": 0.275632, + "nauc_precision_at_10_max": 0.305842, + "nauc_precision_at_10_std": 0.044233, + "nauc_precision_at_10_diff1": 0.09588, + "nauc_precision_at_20_max": 0.200377, + "nauc_precision_at_20_std": 0.153963, + "nauc_precision_at_20_diff1": -0.065507, + "nauc_precision_at_100_max": 0.045865, + "nauc_precision_at_100_std": 0.297418, + "nauc_precision_at_100_diff1": -0.236899, + "nauc_precision_at_1000_max": 0.017265, + "nauc_precision_at_1000_std": 0.299901, + "nauc_precision_at_1000_diff1": -0.26962, + "nauc_mrr_at_1_max": 0.468811, + "nauc_mrr_at_1_std": -0.283114, + "nauc_mrr_at_1_diff1": 0.756843, + "nauc_mrr_at_3_max": 0.498328, + "nauc_mrr_at_3_std": -0.322651, + "nauc_mrr_at_3_diff1": 0.743054, + "nauc_mrr_at_5_max": 0.500984, + "nauc_mrr_at_5_std": -0.315497, + "nauc_mrr_at_5_diff1": 0.743551, + "nauc_mrr_at_10_max": 0.503532, + "nauc_mrr_at_10_std": -0.311268, + "nauc_mrr_at_10_diff1": 0.74609, + "nauc_mrr_at_20_max": 0.501746, + "nauc_mrr_at_20_std": -0.311534, + "nauc_mrr_at_20_diff1": 0.746153, + "nauc_mrr_at_100_max": 0.500552, + "nauc_mrr_at_100_std": -0.311385, + "nauc_mrr_at_100_diff1": 0.746296, + "nauc_mrr_at_1000_max": 0.500506, + "nauc_mrr_at_1000_std": -0.311391, + "nauc_mrr_at_1000_diff1": 0.746308, + "hit_rate_at_1": 0.71093, + "hit_rate_at_3": 0.86823, + "hit_rate_at_5": 0.90807, + "hit_rate_at_10": 0.94382, + "hit_rate_at_20": 0.96936, + "hit_rate_at_100": 0.99591, + "hit_rate_at_1000": 1.0, + "main_score": 0.82735, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.69597, + "ndcg_at_3": 0.76882, + "ndcg_at_5": 0.78496, + "ndcg_at_10": 0.8001, + "ndcg_at_20": 0.8067, + "ndcg_at_100": 0.81731, + "ndcg_at_1000": 0.81989, + "map_at_1": 0.67532, + "map_at_3": 0.74891, + "map_at_5": 0.75848, + "map_at_10": 0.76529, + "map_at_20": 0.76725, + "map_at_100": 0.7688, + "map_at_1000": 0.76894, + "recall_at_1": 0.67532, + "recall_at_3": 0.81956, + "recall_at_5": 0.85699, + "recall_at_10": 0.90148, + "recall_at_20": 0.92602, + "recall_at_100": 0.9819, + "recall_at_1000": 1.0, + "accuracy": 0.67532, + "precision_at_1": 0.69597, + "precision_at_3": 0.28743, + "precision_at_5": 0.18114, + "precision_at_10": 0.09576, + "precision_at_20": 0.04936, + "precision_at_100": 0.01052, + "precision_at_1000": 0.00107, + "mrr_at_1": 0.695975, + "mrr_at_3": 0.75565, + "mrr_at_5": 0.764124, + "mrr_at_10": 0.770048, + "mrr_at_20": 0.771847, + "mrr_at_100": 0.773159, + "mrr_at_1000": 0.773263, + "nauc_ndcg_at_1_max": 0.525298, + "nauc_ndcg_at_1_std": -0.377956, + "nauc_ndcg_at_1_diff1": 0.769185, + "nauc_ndcg_at_3_max": 0.531194, + "nauc_ndcg_at_3_std": -0.493065, + "nauc_ndcg_at_3_diff1": 0.745215, + "nauc_ndcg_at_5_max": 0.53873, + "nauc_ndcg_at_5_std": -0.507111, + "nauc_ndcg_at_5_diff1": 0.738507, + "nauc_ndcg_at_10_max": 0.532007, + "nauc_ndcg_at_10_std": -0.500327, + "nauc_ndcg_at_10_diff1": 0.734715, + "nauc_ndcg_at_20_max": 0.53121, + "nauc_ndcg_at_20_std": -0.505684, + "nauc_ndcg_at_20_diff1": 0.734437, + "nauc_ndcg_at_100_max": 0.534029, + "nauc_ndcg_at_100_std": -0.477701, + "nauc_ndcg_at_100_diff1": 0.742115, + "nauc_ndcg_at_1000_max": 0.53233, + "nauc_ndcg_at_1000_std": -0.477734, + "nauc_ndcg_at_1000_diff1": 0.743707, + "nauc_map_at_1_max": 0.471912, + "nauc_map_at_1_std": -0.368593, + "nauc_map_at_1_diff1": 0.755631, + "nauc_map_at_3_max": 0.522459, + "nauc_map_at_3_std": -0.460428, + "nauc_map_at_3_diff1": 0.746194, + "nauc_map_at_5_max": 0.528621, + "nauc_map_at_5_std": -0.46478, + "nauc_map_at_5_diff1": 0.742761, + "nauc_map_at_10_max": 0.52535, + "nauc_map_at_10_std": -0.462014, + "nauc_map_at_10_diff1": 0.741278, + "nauc_map_at_20_max": 0.524887, + "nauc_map_at_20_std": -0.463215, + "nauc_map_at_20_diff1": 0.741288, + "nauc_map_at_100_max": 0.525301, + "nauc_map_at_100_std": -0.460504, + "nauc_map_at_100_diff1": 0.742611, + "nauc_map_at_1000_max": 0.525271, + "nauc_map_at_1000_std": -0.460488, + "nauc_map_at_1000_diff1": 0.74271, + "nauc_recall_at_1_max": 0.471912, + "nauc_recall_at_1_std": -0.368593, + "nauc_recall_at_1_diff1": 0.755631, + "nauc_recall_at_3_max": 0.536298, + "nauc_recall_at_3_std": -0.60797, + "nauc_recall_at_3_diff1": 0.71951, + "nauc_recall_at_5_max": 0.559477, + "nauc_recall_at_5_std": -0.688341, + "nauc_recall_at_5_diff1": 0.693343, + "nauc_recall_at_10_max": 0.535051, + "nauc_recall_at_10_std": -0.719904, + "nauc_recall_at_10_diff1": 0.658588, + "nauc_recall_at_20_max": 0.528704, + "nauc_recall_at_20_std": -0.844599, + "nauc_recall_at_20_diff1": 0.631626, + "nauc_recall_at_100_max": 0.65305, + "nauc_recall_at_100_std": -0.46891, + "nauc_recall_at_100_diff1": 0.64422, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.525298, + "nauc_precision_at_1_std": -0.377956, + "nauc_precision_at_1_diff1": 0.769185, + "nauc_precision_at_3_max": 0.470903, + "nauc_precision_at_3_std": -0.456679, + "nauc_precision_at_3_diff1": 0.484779, + "nauc_precision_at_5_max": 0.448915, + "nauc_precision_at_5_std": -0.435784, + "nauc_precision_at_5_diff1": 0.378179, + "nauc_precision_at_10_max": 0.341469, + "nauc_precision_at_10_std": -0.354603, + "nauc_precision_at_10_diff1": 0.223504, + "nauc_precision_at_20_max": 0.282674, + "nauc_precision_at_20_std": -0.320289, + "nauc_precision_at_20_diff1": 0.118959, + "nauc_precision_at_100_max": 0.124898, + "nauc_precision_at_100_std": 0.05284, + "nauc_precision_at_100_diff1": -0.138316, + "nauc_precision_at_1000_max": 0.036913, + "nauc_precision_at_1000_std": 0.147202, + "nauc_precision_at_1000_diff1": -0.24452, + "nauc_mrr_at_1_max": 0.525298, + "nauc_mrr_at_1_std": -0.377956, + "nauc_mrr_at_1_diff1": 0.769185, + "nauc_mrr_at_3_max": 0.541448, + "nauc_mrr_at_3_std": -0.461201, + "nauc_mrr_at_3_diff1": 0.759881, + "nauc_mrr_at_5_max": 0.541345, + "nauc_mrr_at_5_std": -0.469251, + "nauc_mrr_at_5_diff1": 0.757422, + "nauc_mrr_at_10_max": 0.539642, + "nauc_mrr_at_10_std": -0.463936, + "nauc_mrr_at_10_diff1": 0.756627, + "nauc_mrr_at_20_max": 0.53977, + "nauc_mrr_at_20_std": -0.464077, + "nauc_mrr_at_20_diff1": 0.756601, + "nauc_mrr_at_100_max": 0.539952, + "nauc_mrr_at_100_std": -0.461286, + "nauc_mrr_at_100_diff1": 0.757388, + "nauc_mrr_at_1000_max": 0.539839, + "nauc_mrr_at_1000_std": -0.461373, + "nauc_mrr_at_1000_diff1": 0.757404, + "hit_rate_at_1": 0.69597, + "hit_rate_at_3": 0.82839, + "hit_rate_at_5": 0.86441, + "hit_rate_at_10": 0.90678, + "hit_rate_at_20": 0.93114, + "hit_rate_at_100": 0.98411, + "hit_rate_at_1000": 1.0, + "main_score": 0.8001, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ] + }, + { + "ndcg_at_1": 0.58463, + "ndcg_at_3": 0.64989, + "ndcg_at_5": 0.6734, + "ndcg_at_10": 0.69148, + "ndcg_at_20": 0.70307, + "ndcg_at_100": 0.72042, + "ndcg_at_1000": 0.72817, + "map_at_1": 0.56386, + "map_at_3": 0.62856, + "map_at_5": 0.6418, + "map_at_10": 0.64964, + "map_at_20": 0.6531, + "map_at_100": 0.65555, + "map_at_1000": 0.65593, + "recall_at_1": 0.56386, + "recall_at_3": 0.69851, + "recall_at_5": 0.75511, + "recall_at_10": 0.80893, + "recall_at_20": 0.85254, + "recall_at_100": 0.94427, + "recall_at_1000": 1.0, + "accuracy": 0.56386, + "precision_at_1": 0.58463, + "precision_at_3": 0.24368, + "precision_at_5": 0.15826, + "precision_at_10": 0.08515, + "precision_at_20": 0.04512, + "precision_at_100": 0.0101, + "precision_at_1000": 0.00108, + "mrr_at_1": 0.584631, + "mrr_at_3": 0.64261, + "mrr_at_5": 0.655279, + "mrr_at_10": 0.662668, + "mrr_at_20": 0.665737, + "mrr_at_100": 0.667772, + "mrr_at_1000": 0.668071, + "nauc_ndcg_at_1_max": 0.491986, + "nauc_ndcg_at_1_std": -0.37036, + "nauc_ndcg_at_1_diff1": 0.773668, + "nauc_ndcg_at_3_max": 0.501732, + "nauc_ndcg_at_3_std": -0.394958, + "nauc_ndcg_at_3_diff1": 0.720611, + "nauc_ndcg_at_5_max": 0.508439, + "nauc_ndcg_at_5_std": -0.412909, + "nauc_ndcg_at_5_diff1": 0.718169, + "nauc_ndcg_at_10_max": 0.510297, + "nauc_ndcg_at_10_std": -0.415113, + "nauc_ndcg_at_10_diff1": 0.717072, + "nauc_ndcg_at_20_max": 0.515358, + "nauc_ndcg_at_20_std": -0.410946, + "nauc_ndcg_at_20_diff1": 0.717909, + "nauc_ndcg_at_100_max": 0.511811, + "nauc_ndcg_at_100_std": -0.407438, + "nauc_ndcg_at_100_diff1": 0.725904, + "nauc_ndcg_at_1000_max": 0.507354, + "nauc_ndcg_at_1000_std": -0.403293, + "nauc_ndcg_at_1000_diff1": 0.728015, + "nauc_map_at_1_max": 0.465715, + "nauc_map_at_1_std": -0.352262, + "nauc_map_at_1_diff1": 0.760523, + "nauc_map_at_3_max": 0.494, + "nauc_map_at_3_std": -0.388299, + "nauc_map_at_3_diff1": 0.730215, + "nauc_map_at_5_max": 0.497506, + "nauc_map_at_5_std": -0.397649, + "nauc_map_at_5_diff1": 0.729528, + "nauc_map_at_10_max": 0.498896, + "nauc_map_at_10_std": -0.398304, + "nauc_map_at_10_diff1": 0.729434, + "nauc_map_at_20_max": 0.50031, + "nauc_map_at_20_std": -0.396928, + "nauc_map_at_20_diff1": 0.72968, + "nauc_map_at_100_max": 0.499478, + "nauc_map_at_100_std": -0.396742, + "nauc_map_at_100_diff1": 0.730931, + "nauc_map_at_1000_max": 0.499283, + "nauc_map_at_1000_std": -0.396572, + "nauc_map_at_1000_diff1": 0.731018, + "nauc_recall_at_1_max": 0.465715, + "nauc_recall_at_1_std": -0.352262, + "nauc_recall_at_1_diff1": 0.760523, + "nauc_recall_at_3_max": 0.506143, + "nauc_recall_at_3_std": -0.411311, + "nauc_recall_at_3_diff1": 0.670046, + "nauc_recall_at_5_max": 0.52428, + "nauc_recall_at_5_std": -0.464914, + "nauc_recall_at_5_diff1": 0.649711, + "nauc_recall_at_10_max": 0.531039, + "nauc_recall_at_10_std": -0.486712, + "nauc_recall_at_10_diff1": 0.627333, + "nauc_recall_at_20_max": 0.571805, + "nauc_recall_at_20_std": -0.478588, + "nauc_recall_at_20_diff1": 0.611786, + "nauc_recall_at_100_max": 0.625923, + "nauc_recall_at_100_std": -0.525749, + "nauc_recall_at_100_diff1": 0.65027, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.491986, + "nauc_precision_at_1_std": -0.37036, + "nauc_precision_at_1_diff1": 0.773668, + "nauc_precision_at_3_max": 0.499941, + "nauc_precision_at_3_std": -0.398303, + "nauc_precision_at_3_diff1": 0.607238, + "nauc_precision_at_5_max": 0.499244, + "nauc_precision_at_5_std": -0.43046, + "nauc_precision_at_5_diff1": 0.558473, + "nauc_precision_at_10_max": 0.479763, + "nauc_precision_at_10_std": -0.404302, + "nauc_precision_at_10_diff1": 0.484189, + "nauc_precision_at_20_max": 0.453224, + "nauc_precision_at_20_std": -0.330278, + "nauc_precision_at_20_diff1": 0.388492, + "nauc_precision_at_100_max": 0.236603, + "nauc_precision_at_100_std": -0.120985, + "nauc_precision_at_100_diff1": 0.128784, + "nauc_precision_at_1000_max": -0.03547, + "nauc_precision_at_1000_std": 0.108715, + "nauc_precision_at_1000_diff1": -0.168865, + "nauc_mrr_at_1_max": 0.491986, + "nauc_mrr_at_1_std": -0.37036, + "nauc_mrr_at_1_diff1": 0.773668, + "nauc_mrr_at_3_max": 0.507572, + "nauc_mrr_at_3_std": -0.389319, + "nauc_mrr_at_3_diff1": 0.740776, + "nauc_mrr_at_5_max": 0.510605, + "nauc_mrr_at_5_std": -0.398221, + "nauc_mrr_at_5_diff1": 0.739761, + "nauc_mrr_at_10_max": 0.510121, + "nauc_mrr_at_10_std": -0.398608, + "nauc_mrr_at_10_diff1": 0.739765, + "nauc_mrr_at_20_max": 0.510865, + "nauc_mrr_at_20_std": -0.398393, + "nauc_mrr_at_20_diff1": 0.74039, + "nauc_mrr_at_100_max": 0.510154, + "nauc_mrr_at_100_std": -0.397646, + "nauc_mrr_at_100_diff1": 0.741037, + "nauc_mrr_at_1000_max": 0.509985, + "nauc_mrr_at_1000_std": -0.397541, + "nauc_mrr_at_1000_diff1": 0.741099, + "hit_rate_at_1": 0.58463, + "hit_rate_at_3": 0.7134, + "hit_rate_at_5": 0.76947, + "hit_rate_at_10": 0.82347, + "hit_rate_at_20": 0.86604, + "hit_rate_at_100": 0.95119, + "hit_rate_at_1000": 1.0, + "main_score": 0.69148, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.7837, + "ndcg_at_3": 0.84659, + "ndcg_at_5": 0.86378, + "ndcg_at_10": 0.87489, + "ndcg_at_20": 0.88029, + "ndcg_at_100": 0.88306, + "ndcg_at_1000": 0.88334, + "map_at_1": 0.764, + "map_at_3": 0.82829, + "map_at_5": 0.83877, + "map_at_10": 0.84368, + "map_at_20": 0.84536, + "map_at_100": 0.84588, + "map_at_1000": 0.84589, + "recall_at_1": 0.764, + "recall_at_3": 0.89118, + "recall_at_5": 0.93184, + "recall_at_10": 0.9647, + "recall_at_20": 0.98466, + "recall_at_100": 0.99799, + "recall_at_1000": 1.0, + "accuracy": 0.764, + "precision_at_1": 0.7837, + "precision_at_3": 0.3112, + "precision_at_5": 0.19618, + "precision_at_10": 0.10181, + "precision_at_20": 0.05211, + "precision_at_100": 0.01058, + "precision_at_1000": 0.00106, + "mrr_at_1": 0.783702, + "mrr_at_3": 0.836016, + "mrr_at_5": 0.844567, + "mrr_at_10": 0.848967, + "mrr_at_20": 0.850431, + "mrr_at_100": 0.850796, + "mrr_at_1000": 0.850811, + "nauc_ndcg_at_1_max": 0.503554, + "nauc_ndcg_at_1_std": -0.399734, + "nauc_ndcg_at_1_diff1": 0.808915, + "nauc_ndcg_at_3_max": 0.516283, + "nauc_ndcg_at_3_std": -0.429496, + "nauc_ndcg_at_3_diff1": 0.768185, + "nauc_ndcg_at_5_max": 0.527838, + "nauc_ndcg_at_5_std": -0.413632, + "nauc_ndcg_at_5_diff1": 0.770005, + "nauc_ndcg_at_10_max": 0.530049, + "nauc_ndcg_at_10_std": -0.401676, + "nauc_ndcg_at_10_diff1": 0.778661, + "nauc_ndcg_at_20_max": 0.527802, + "nauc_ndcg_at_20_std": -0.405462, + "nauc_ndcg_at_20_diff1": 0.779188, + "nauc_ndcg_at_100_max": 0.524538, + "nauc_ndcg_at_100_std": -0.405244, + "nauc_ndcg_at_100_diff1": 0.78373, + "nauc_ndcg_at_1000_max": 0.52387, + "nauc_ndcg_at_1000_std": -0.406239, + "nauc_ndcg_at_1000_diff1": 0.783335, + "nauc_map_at_1_max": 0.46485, + "nauc_map_at_1_std": -0.405618, + "nauc_map_at_1_diff1": 0.799319, + "nauc_map_at_3_max": 0.508852, + "nauc_map_at_3_std": -0.429707, + "nauc_map_at_3_diff1": 0.781178, + "nauc_map_at_5_max": 0.51681, + "nauc_map_at_5_std": -0.418292, + "nauc_map_at_5_diff1": 0.781878, + "nauc_map_at_10_max": 0.517632, + "nauc_map_at_10_std": -0.413057, + "nauc_map_at_10_diff1": 0.785201, + "nauc_map_at_20_max": 0.517854, + "nauc_map_at_20_std": -0.413051, + "nauc_map_at_20_diff1": 0.785393, + "nauc_map_at_100_max": 0.51743, + "nauc_map_at_100_std": -0.413035, + "nauc_map_at_100_diff1": 0.786125, + "nauc_map_at_1000_max": 0.517401, + "nauc_map_at_1000_std": -0.413076, + "nauc_map_at_1000_diff1": 0.786109, + "nauc_recall_at_1_max": 0.46485, + "nauc_recall_at_1_std": -0.405618, + "nauc_recall_at_1_diff1": 0.799319, + "nauc_recall_at_3_max": 0.526979, + "nauc_recall_at_3_std": -0.461407, + "nauc_recall_at_3_diff1": 0.713535, + "nauc_recall_at_5_max": 0.571668, + "nauc_recall_at_5_std": -0.409569, + "nauc_recall_at_5_diff1": 0.687657, + "nauc_recall_at_10_max": 0.626484, + "nauc_recall_at_10_std": -0.31508, + "nauc_recall_at_10_diff1": 0.701259, + "nauc_recall_at_20_max": 0.66759, + "nauc_recall_at_20_std": -0.344582, + "nauc_recall_at_20_diff1": 0.629713, + "nauc_recall_at_100_max": 0.777112, + "nauc_recall_at_100_std": -0.02419, + "nauc_recall_at_100_diff1": 0.93467, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.503554, + "nauc_precision_at_1_std": -0.399734, + "nauc_precision_at_1_diff1": 0.808915, + "nauc_precision_at_3_max": 0.365687, + "nauc_precision_at_3_std": -0.232212, + "nauc_precision_at_3_diff1": 0.337781, + "nauc_precision_at_5_max": 0.296114, + "nauc_precision_at_5_std": -0.064293, + "nauc_precision_at_5_diff1": 0.136655, + "nauc_precision_at_10_max": 0.197298, + "nauc_precision_at_10_std": 0.113734, + "nauc_precision_at_10_diff1": -0.051763, + "nauc_precision_at_20_max": 0.104157, + "nauc_precision_at_20_std": 0.215126, + "nauc_precision_at_20_diff1": -0.222077, + "nauc_precision_at_100_max": 0.005407, + "nauc_precision_at_100_std": 0.280486, + "nauc_precision_at_100_diff1": -0.315816, + "nauc_precision_at_1000_max": -0.009361, + "nauc_precision_at_1000_std": 0.286317, + "nauc_precision_at_1000_diff1": -0.339747, + "nauc_mrr_at_1_max": 0.503554, + "nauc_mrr_at_1_std": -0.399734, + "nauc_mrr_at_1_diff1": 0.808915, + "nauc_mrr_at_3_max": 0.520167, + "nauc_mrr_at_3_std": -0.405666, + "nauc_mrr_at_3_diff1": 0.779308, + "nauc_mrr_at_5_max": 0.523299, + "nauc_mrr_at_5_std": -0.400792, + "nauc_mrr_at_5_diff1": 0.782338, + "nauc_mrr_at_10_max": 0.524077, + "nauc_mrr_at_10_std": -0.397625, + "nauc_mrr_at_10_diff1": 0.786106, + "nauc_mrr_at_20_max": 0.522724, + "nauc_mrr_at_20_std": -0.399671, + "nauc_mrr_at_20_diff1": 0.7862, + "nauc_mrr_at_100_max": 0.522743, + "nauc_mrr_at_100_std": -0.399505, + "nauc_mrr_at_100_diff1": 0.786969, + "nauc_mrr_at_1000_max": 0.522714, + "nauc_mrr_at_1000_std": -0.399545, + "nauc_mrr_at_1000_diff1": 0.786953, + "hit_rate_at_1": 0.7837, + "hit_rate_at_3": 0.8994, + "hit_rate_at_5": 0.93662, + "hit_rate_at_10": 0.96781, + "hit_rate_at_20": 0.98692, + "hit_rate_at_100": 0.99799, + "hit_rate_at_1000": 1.0, + "main_score": 0.87489, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ] + }, + { + "ndcg_at_1": 0.42872, + "ndcg_at_3": 0.50504, + "ndcg_at_5": 0.53725, + "ndcg_at_10": 0.56413, + "ndcg_at_20": 0.58044, + "ndcg_at_100": 0.60399, + "ndcg_at_1000": 0.61712, + "map_at_1": 0.40617, + "map_at_3": 0.47904, + "map_at_5": 0.4981, + "map_at_10": 0.50961, + "map_at_20": 0.51447, + "map_at_100": 0.51803, + "map_at_1000": 0.51859, + "recall_at_1": 0.40617, + "recall_at_3": 0.56122, + "recall_at_5": 0.63718, + "recall_at_10": 0.718, + "recall_at_20": 0.78061, + "recall_at_100": 0.90236, + "recall_at_1000": 1.0, + "accuracy": 0.40617, + "precision_at_1": 0.42872, + "precision_at_3": 0.20083, + "precision_at_5": 0.13798, + "precision_at_10": 0.07815, + "precision_at_20": 0.04272, + "precision_at_100": 0.00994, + "precision_at_1000": 0.00111, + "mrr_at_1": 0.42872, + "mrr_at_3": 0.499133, + "mrr_at_5": 0.515314, + "mrr_at_10": 0.525918, + "mrr_at_20": 0.529741, + "mrr_at_100": 0.532778, + "mrr_at_1000": 0.533263, + "nauc_ndcg_at_1_max": 0.418125, + "nauc_ndcg_at_1_std": 0.010701, + "nauc_ndcg_at_1_diff1": 0.600219, + "nauc_ndcg_at_3_max": 0.43159, + "nauc_ndcg_at_3_std": -0.010012, + "nauc_ndcg_at_3_diff1": 0.562404, + "nauc_ndcg_at_5_max": 0.438071, + "nauc_ndcg_at_5_std": 0.009191, + "nauc_ndcg_at_5_diff1": 0.5502, + "nauc_ndcg_at_10_max": 0.440912, + "nauc_ndcg_at_10_std": 0.014952, + "nauc_ndcg_at_10_diff1": 0.546848, + "nauc_ndcg_at_20_max": 0.443643, + "nauc_ndcg_at_20_std": 0.022807, + "nauc_ndcg_at_20_diff1": 0.551222, + "nauc_ndcg_at_100_max": 0.440944, + "nauc_ndcg_at_100_std": 0.022451, + "nauc_ndcg_at_100_diff1": 0.554666, + "nauc_ndcg_at_1000_max": 0.438187, + "nauc_ndcg_at_1000_std": 0.016189, + "nauc_ndcg_at_1000_diff1": 0.559142, + "nauc_map_at_1_max": 0.395545, + "nauc_map_at_1_std": 0.006268, + "nauc_map_at_1_diff1": 0.611879, + "nauc_map_at_3_max": 0.424035, + "nauc_map_at_3_std": -0.00665, + "nauc_map_at_3_diff1": 0.573844, + "nauc_map_at_5_max": 0.429202, + "nauc_map_at_5_std": 0.005145, + "nauc_map_at_5_diff1": 0.566787, + "nauc_map_at_10_max": 0.429864, + "nauc_map_at_10_std": 0.007788, + "nauc_map_at_10_diff1": 0.565212, + "nauc_map_at_20_max": 0.430691, + "nauc_map_at_20_std": 0.009567, + "nauc_map_at_20_diff1": 0.566804, + "nauc_map_at_100_max": 0.430196, + "nauc_map_at_100_std": 0.009335, + "nauc_map_at_100_diff1": 0.567249, + "nauc_map_at_1000_max": 0.43019, + "nauc_map_at_1000_std": 0.009202, + "nauc_map_at_1000_diff1": 0.567433, + "nauc_recall_at_1_max": 0.395545, + "nauc_recall_at_1_std": 0.006268, + "nauc_recall_at_1_diff1": 0.611879, + "nauc_recall_at_3_max": 0.438418, + "nauc_recall_at_3_std": -0.028411, + "nauc_recall_at_3_diff1": 0.523474, + "nauc_recall_at_5_max": 0.45173, + "nauc_recall_at_5_std": 0.020732, + "nauc_recall_at_5_diff1": 0.483975, + "nauc_recall_at_10_max": 0.462144, + "nauc_recall_at_10_std": 0.040926, + "nauc_recall_at_10_diff1": 0.458131, + "nauc_recall_at_20_max": 0.486587, + "nauc_recall_at_20_std": 0.092645, + "nauc_recall_at_20_diff1": 0.469183, + "nauc_recall_at_100_max": 0.495877, + "nauc_recall_at_100_std": 0.160038, + "nauc_recall_at_100_diff1": 0.446367, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.418125, + "nauc_precision_at_1_std": 0.010701, + "nauc_precision_at_1_diff1": 0.600219, + "nauc_precision_at_3_max": 0.45635, + "nauc_precision_at_3_std": -0.015586, + "nauc_precision_at_3_diff1": 0.478738, + "nauc_precision_at_5_max": 0.441736, + "nauc_precision_at_5_std": 0.034196, + "nauc_precision_at_5_diff1": 0.395345, + "nauc_precision_at_10_max": 0.421632, + "nauc_precision_at_10_std": 0.064473, + "nauc_precision_at_10_diff1": 0.329586, + "nauc_precision_at_20_max": 0.382027, + "nauc_precision_at_20_std": 0.100501, + "nauc_precision_at_20_diff1": 0.277834, + "nauc_precision_at_100_max": 0.252946, + "nauc_precision_at_100_std": 0.130406, + "nauc_precision_at_100_diff1": 0.101485, + "nauc_precision_at_1000_max": 0.023751, + "nauc_precision_at_1000_std": 0.054771, + "nauc_precision_at_1000_diff1": -0.148504, + "nauc_mrr_at_1_max": 0.418125, + "nauc_mrr_at_1_std": 0.010701, + "nauc_mrr_at_1_diff1": 0.600219, + "nauc_mrr_at_3_max": 0.435495, + "nauc_mrr_at_3_std": 0.00154, + "nauc_mrr_at_3_diff1": 0.570812, + "nauc_mrr_at_5_max": 0.436049, + "nauc_mrr_at_5_std": 0.009069, + "nauc_mrr_at_5_diff1": 0.565797, + "nauc_mrr_at_10_max": 0.437597, + "nauc_mrr_at_10_std": 0.010705, + "nauc_mrr_at_10_diff1": 0.564968, + "nauc_mrr_at_20_max": 0.437911, + "nauc_mrr_at_20_std": 0.012887, + "nauc_mrr_at_20_diff1": 0.56529, + "nauc_mrr_at_100_max": 0.437341, + "nauc_mrr_at_100_std": 0.012483, + "nauc_mrr_at_100_diff1": 0.565726, + "nauc_mrr_at_1000_max": 0.437286, + "nauc_mrr_at_1000_std": 0.012342, + "nauc_mrr_at_1000_diff1": 0.565924, + "hit_rate_at_1": 0.42872, + "hit_rate_at_3": 0.58585, + "hit_rate_at_5": 0.65765, + "hit_rate_at_10": 0.73673, + "hit_rate_at_20": 0.79396, + "hit_rate_at_100": 0.90843, + "hit_rate_at_1000": 1.0, + "main_score": 0.56413, + "hf_subset": "sw", + "languages": [ + "swa-Latn" + ] + }, + { + "ndcg_at_1": 0.70333, + "ndcg_at_3": 0.78466, + "ndcg_at_5": 0.80616, + "ndcg_at_10": 0.82005, + "ndcg_at_20": 0.82897, + "ndcg_at_100": 0.83442, + "ndcg_at_1000": 0.83502, + "map_at_1": 0.68352, + "map_at_3": 0.76343, + "map_at_5": 0.77579, + "map_at_10": 0.78178, + "map_at_20": 0.78434, + "map_at_100": 0.78528, + "map_at_1000": 0.78531, + "recall_at_1": 0.68352, + "recall_at_3": 0.84111, + "recall_at_5": 0.89222, + "recall_at_10": 0.93426, + "recall_at_20": 0.96907, + "recall_at_100": 0.99611, + "recall_at_1000": 1.0, + "accuracy": 0.68352, + "precision_at_1": 0.70333, + "precision_at_3": 0.29407, + "precision_at_5": 0.18733, + "precision_at_10": 0.09833, + "precision_at_20": 0.051, + "precision_at_100": 0.01048, + "precision_at_1000": 0.00105, + "mrr_at_1": 0.703333, + "mrr_at_3": 0.768148, + "mrr_at_5": 0.779704, + "mrr_at_10": 0.78503, + "mrr_at_20": 0.787354, + "mrr_at_100": 0.788237, + "mrr_at_1000": 0.788263, + "nauc_ndcg_at_1_max": 0.369295, + "nauc_ndcg_at_1_std": -0.283611, + "nauc_ndcg_at_1_diff1": 0.760955, + "nauc_ndcg_at_3_max": 0.407864, + "nauc_ndcg_at_3_std": -0.332439, + "nauc_ndcg_at_3_diff1": 0.721979, + "nauc_ndcg_at_5_max": 0.405442, + "nauc_ndcg_at_5_std": -0.335494, + "nauc_ndcg_at_5_diff1": 0.716671, + "nauc_ndcg_at_10_max": 0.41259, + "nauc_ndcg_at_10_std": -0.332198, + "nauc_ndcg_at_10_diff1": 0.721082, + "nauc_ndcg_at_20_max": 0.412603, + "nauc_ndcg_at_20_std": -0.316212, + "nauc_ndcg_at_20_diff1": 0.726733, + "nauc_ndcg_at_100_max": 0.403166, + "nauc_ndcg_at_100_std": -0.313897, + "nauc_ndcg_at_100_diff1": 0.72905, + "nauc_ndcg_at_1000_max": 0.400895, + "nauc_ndcg_at_1000_std": -0.316468, + "nauc_ndcg_at_1000_diff1": 0.728936, + "nauc_map_at_1_max": 0.344936, + "nauc_map_at_1_std": -0.272411, + "nauc_map_at_1_diff1": 0.76158, + "nauc_map_at_3_max": 0.393162, + "nauc_map_at_3_std": -0.319535, + "nauc_map_at_3_diff1": 0.733361, + "nauc_map_at_5_max": 0.391884, + "nauc_map_at_5_std": -0.320822, + "nauc_map_at_5_diff1": 0.730788, + "nauc_map_at_10_max": 0.394814, + "nauc_map_at_10_std": -0.318282, + "nauc_map_at_10_diff1": 0.732454, + "nauc_map_at_20_max": 0.394639, + "nauc_map_at_20_std": -0.314481, + "nauc_map_at_20_diff1": 0.733979, + "nauc_map_at_100_max": 0.393379, + "nauc_map_at_100_std": -0.313931, + "nauc_map_at_100_diff1": 0.734106, + "nauc_map_at_1000_max": 0.393268, + "nauc_map_at_1000_std": -0.31407, + "nauc_map_at_1000_diff1": 0.734104, + "nauc_recall_at_1_max": 0.344936, + "nauc_recall_at_1_std": -0.272411, + "nauc_recall_at_1_diff1": 0.76158, + "nauc_recall_at_3_max": 0.454478, + "nauc_recall_at_3_std": -0.383894, + "nauc_recall_at_3_diff1": 0.679417, + "nauc_recall_at_5_max": 0.467701, + "nauc_recall_at_5_std": -0.417654, + "nauc_recall_at_5_diff1": 0.64627, + "nauc_recall_at_10_max": 0.552182, + "nauc_recall_at_10_std": -0.449845, + "nauc_recall_at_10_diff1": 0.637607, + "nauc_recall_at_20_max": 0.710364, + "nauc_recall_at_20_std": -0.268049, + "nauc_recall_at_20_diff1": 0.656074, + "nauc_recall_at_100_max": 0.960317, + "nauc_recall_at_100_std": 0.29085, + "nauc_recall_at_100_diff1": 0.754235, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.369295, + "nauc_precision_at_1_std": -0.283611, + "nauc_precision_at_1_diff1": 0.760955, + "nauc_precision_at_3_max": 0.349863, + "nauc_precision_at_3_std": -0.27806, + "nauc_precision_at_3_diff1": 0.404514, + "nauc_precision_at_5_max": 0.309676, + "nauc_precision_at_5_std": -0.261687, + "nauc_precision_at_5_diff1": 0.279249, + "nauc_precision_at_10_max": 0.268993, + "nauc_precision_at_10_std": -0.192874, + "nauc_precision_at_10_diff1": 0.122281, + "nauc_precision_at_20_max": 0.205006, + "nauc_precision_at_20_std": -0.039051, + "nauc_precision_at_20_diff1": -0.041632, + "nauc_precision_at_100_max": 0.056536, + "nauc_precision_at_100_std": 0.046788, + "nauc_precision_at_100_diff1": -0.248937, + "nauc_precision_at_1000_max": -0.00033, + "nauc_precision_at_1000_std": 0.026725, + "nauc_precision_at_1000_diff1": -0.291667, + "nauc_mrr_at_1_max": 0.369295, + "nauc_mrr_at_1_std": -0.283611, + "nauc_mrr_at_1_diff1": 0.760955, + "nauc_mrr_at_3_max": 0.401632, + "nauc_mrr_at_3_std": -0.316917, + "nauc_mrr_at_3_diff1": 0.730244, + "nauc_mrr_at_5_max": 0.397678, + "nauc_mrr_at_5_std": -0.316315, + "nauc_mrr_at_5_diff1": 0.728033, + "nauc_mrr_at_10_max": 0.399497, + "nauc_mrr_at_10_std": -0.315805, + "nauc_mrr_at_10_diff1": 0.730059, + "nauc_mrr_at_20_max": 0.399376, + "nauc_mrr_at_20_std": -0.312742, + "nauc_mrr_at_20_diff1": 0.731354, + "nauc_mrr_at_100_max": 0.39824, + "nauc_mrr_at_100_std": -0.311938, + "nauc_mrr_at_100_diff1": 0.731597, + "nauc_mrr_at_1000_max": 0.398158, + "nauc_mrr_at_1000_std": -0.312026, + "nauc_mrr_at_1000_diff1": 0.731583, + "hit_rate_at_1": 0.70333, + "hit_rate_at_3": 0.84667, + "hit_rate_at_5": 0.89667, + "hit_rate_at_10": 0.93667, + "hit_rate_at_20": 0.97, + "hit_rate_at_100": 0.99667, + "hit_rate_at_1000": 1.0, + "main_score": 0.82005, + "hf_subset": "hi", + "languages": [ + "hin-Deva" + ] + }, + { + "ndcg_at_1": 0.62308, + "ndcg_at_3": 0.70542, + "ndcg_at_5": 0.73307, + "ndcg_at_10": 0.75041, + "ndcg_at_20": 0.76131, + "ndcg_at_100": 0.77388, + "ndcg_at_1000": 0.77623, + "map_at_1": 0.59874, + "map_at_3": 0.67918, + "map_at_5": 0.69608, + "map_at_10": 0.70427, + "map_at_20": 0.70745, + "map_at_100": 0.70953, + "map_at_1000": 0.70964, + "recall_at_1": 0.59874, + "recall_at_3": 0.7637, + "recall_at_5": 0.8284, + "recall_at_10": 0.87879, + "recall_at_20": 0.9205, + "recall_at_100": 0.98349, + "recall_at_1000": 1.0, + "accuracy": 0.59874, + "precision_at_1": 0.62308, + "precision_at_3": 0.27068, + "precision_at_5": 0.17773, + "precision_at_10": 0.0953, + "precision_at_20": 0.05005, + "precision_at_100": 0.01072, + "precision_at_1000": 0.00109, + "mrr_at_1": 0.623085, + "mrr_at_3": 0.69714, + "mrr_at_5": 0.710163, + "mrr_at_10": 0.716008, + "mrr_at_20": 0.718783, + "mrr_at_100": 0.720556, + "mrr_at_1000": 0.720648, + "nauc_ndcg_at_1_max": 0.381914, + "nauc_ndcg_at_1_std": -0.162657, + "nauc_ndcg_at_1_diff1": 0.691881, + "nauc_ndcg_at_3_max": 0.432961, + "nauc_ndcg_at_3_std": -0.201569, + "nauc_ndcg_at_3_diff1": 0.668726, + "nauc_ndcg_at_5_max": 0.440447, + "nauc_ndcg_at_5_std": -0.19615, + "nauc_ndcg_at_5_diff1": 0.663297, + "nauc_ndcg_at_10_max": 0.450515, + "nauc_ndcg_at_10_std": -0.184958, + "nauc_ndcg_at_10_diff1": 0.663936, + "nauc_ndcg_at_20_max": 0.442815, + "nauc_ndcg_at_20_std": -0.18065, + "nauc_ndcg_at_20_diff1": 0.667897, + "nauc_ndcg_at_100_max": 0.436463, + "nauc_ndcg_at_100_std": -0.180598, + "nauc_ndcg_at_100_diff1": 0.669837, + "nauc_ndcg_at_1000_max": 0.433447, + "nauc_ndcg_at_1000_std": -0.18001, + "nauc_ndcg_at_1000_diff1": 0.67005, + "nauc_map_at_1_max": 0.362181, + "nauc_map_at_1_std": -0.146822, + "nauc_map_at_1_diff1": 0.684012, + "nauc_map_at_3_max": 0.415377, + "nauc_map_at_3_std": -0.188789, + "nauc_map_at_3_diff1": 0.673339, + "nauc_map_at_5_max": 0.420049, + "nauc_map_at_5_std": -0.186412, + "nauc_map_at_5_diff1": 0.670637, + "nauc_map_at_10_max": 0.425039, + "nauc_map_at_10_std": -0.18094, + "nauc_map_at_10_diff1": 0.671129, + "nauc_map_at_20_max": 0.423075, + "nauc_map_at_20_std": -0.179241, + "nauc_map_at_20_diff1": 0.671943, + "nauc_map_at_100_max": 0.422177, + "nauc_map_at_100_std": -0.179131, + "nauc_map_at_100_diff1": 0.67215, + "nauc_map_at_1000_max": 0.42208, + "nauc_map_at_1000_std": -0.179078, + "nauc_map_at_1000_diff1": 0.672178, + "nauc_recall_at_1_max": 0.362181, + "nauc_recall_at_1_std": -0.146822, + "nauc_recall_at_1_diff1": 0.684012, + "nauc_recall_at_3_max": 0.477512, + "nauc_recall_at_3_std": -0.243268, + "nauc_recall_at_3_diff1": 0.646126, + "nauc_recall_at_5_max": 0.511955, + "nauc_recall_at_5_std": -0.236042, + "nauc_recall_at_5_diff1": 0.619896, + "nauc_recall_at_10_max": 0.587281, + "nauc_recall_at_10_std": -0.202184, + "nauc_recall_at_10_diff1": 0.615933, + "nauc_recall_at_20_max": 0.57494, + "nauc_recall_at_20_std": -0.181003, + "nauc_recall_at_20_diff1": 0.635713, + "nauc_recall_at_100_max": 0.701358, + "nauc_recall_at_100_std": -0.194022, + "nauc_recall_at_100_diff1": 0.666832, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.381914, + "nauc_precision_at_1_std": -0.162657, + "nauc_precision_at_1_diff1": 0.691881, + "nauc_precision_at_3_max": 0.455128, + "nauc_precision_at_3_std": -0.209464, + "nauc_precision_at_3_diff1": 0.514636, + "nauc_precision_at_5_max": 0.406831, + "nauc_precision_at_5_std": -0.146076, + "nauc_precision_at_5_diff1": 0.358791, + "nauc_precision_at_10_max": 0.363608, + "nauc_precision_at_10_std": -0.052773, + "nauc_precision_at_10_diff1": 0.214239, + "nauc_precision_at_20_max": 0.278397, + "nauc_precision_at_20_std": 0.015274, + "nauc_precision_at_20_diff1": 0.108628, + "nauc_precision_at_100_max": 0.123435, + "nauc_precision_at_100_std": 0.089963, + "nauc_precision_at_100_diff1": -0.137153, + "nauc_precision_at_1000_max": 0.050409, + "nauc_precision_at_1000_std": 0.13168, + "nauc_precision_at_1000_diff1": -0.214658, + "nauc_mrr_at_1_max": 0.381914, + "nauc_mrr_at_1_std": -0.162657, + "nauc_mrr_at_1_diff1": 0.691881, + "nauc_mrr_at_3_max": 0.426, + "nauc_mrr_at_3_std": -0.191761, + "nauc_mrr_at_3_diff1": 0.675051, + "nauc_mrr_at_5_max": 0.425228, + "nauc_mrr_at_5_std": -0.188143, + "nauc_mrr_at_5_diff1": 0.672214, + "nauc_mrr_at_10_max": 0.4259, + "nauc_mrr_at_10_std": -0.185123, + "nauc_mrr_at_10_diff1": 0.672202, + "nauc_mrr_at_20_max": 0.423962, + "nauc_mrr_at_20_std": -0.184551, + "nauc_mrr_at_20_diff1": 0.673312, + "nauc_mrr_at_100_max": 0.423114, + "nauc_mrr_at_100_std": -0.184441, + "nauc_mrr_at_100_diff1": 0.673653, + "nauc_mrr_at_1000_max": 0.42297, + "nauc_mrr_at_1000_std": -0.184459, + "nauc_mrr_at_1000_diff1": 0.67363, + "hit_rate_at_1": 0.62308, + "hit_rate_at_3": 0.7855, + "hit_rate_at_5": 0.8427, + "hit_rate_at_10": 0.88662, + "hit_rate_at_20": 0.92646, + "hit_rate_at_100": 0.9857, + "hit_rate_at_1000": 1.0, + "main_score": 0.75041, + "hf_subset": "ur", + "languages": [ + "urd-Arab" + ] + } + ] + }, + "evaluation_time": 4623.136050701141, + "kg_co2_emissions": null, + "date": 1777884839.007615 +} \ No newline at end of file diff --git a/results/Octen__Octen-Embedding-0.6B/1a00a4e837bd788f6f8d91bc43201a5e52cf8ef8/LexRetrieval.v1.json b/results/Octen__Octen-Embedding-0.6B/1a00a4e837bd788f6f8d91bc43201a5e52cf8ef8/LexRetrieval.v1.json new file mode 100644 index 0000000000..615e2b7507 --- /dev/null +++ b/results/Octen__Octen-Embedding-0.6B/1a00a4e837bd788f6f8d91bc43201a5e52cf8ef8/LexRetrieval.v1.json @@ -0,0 +1,167 @@ +{ + "dataset_revision": "b5b22dee3a30bb32ac579c14afe9eb152e8c70c7", + "task_name": "LexRetrieval.v1", + "mteb_version": "2.12.38", + "scores": { + "test": [ + { + "ndcg_at_1": 0.91726, + "ndcg_at_3": 0.94203, + "ndcg_at_5": 0.94503, + "ndcg_at_10": 0.94799, + "ndcg_at_20": 0.94952, + "ndcg_at_100": 0.95129, + "ndcg_at_1000": 0.9523, + "map_at_1": 0.91726, + "map_at_3": 0.93643, + "map_at_5": 0.93812, + "map_at_10": 0.93936, + "map_at_20": 0.93979, + "map_at_100": 0.94002, + "map_at_1000": 0.94006, + "recall_at_1": 0.91726, + "recall_at_3": 0.95804, + "recall_at_5": 0.9652, + "recall_at_10": 0.97427, + "recall_at_20": 0.98025, + "recall_at_100": 0.98996, + "recall_at_1000": 0.99786, + "accuracy": 0.91726, + "precision_at_1": 0.91726, + "precision_at_3": 0.31935, + "precision_at_5": 0.19304, + "precision_at_10": 0.09743, + "precision_at_20": 0.04901, + "precision_at_100": 0.0099, + "precision_at_1000": 0.001, + "mrr_at_1": 0.917263, + "mrr_at_3": 0.936426, + "mrr_at_5": 0.938123, + "mrr_at_10": 0.939358, + "mrr_at_20": 0.939786, + "mrr_at_100": 0.940022, + "mrr_at_1000": 0.940061, + "nauc_ndcg_at_1_max": 0.80052, + "nauc_ndcg_at_1_std": 0.215234, + "nauc_ndcg_at_1_diff1": 0.92565, + "nauc_ndcg_at_3_max": 0.868694, + "nauc_ndcg_at_3_std": 0.276488, + "nauc_ndcg_at_3_diff1": 0.936239, + "nauc_ndcg_at_5_max": 0.866319, + "nauc_ndcg_at_5_std": 0.282652, + "nauc_ndcg_at_5_diff1": 0.935371, + "nauc_ndcg_at_10_max": 0.861439, + "nauc_ndcg_at_10_std": 0.280712, + "nauc_ndcg_at_10_diff1": 0.93408, + "nauc_ndcg_at_20_max": 0.859388, + "nauc_ndcg_at_20_std": 0.276634, + "nauc_ndcg_at_20_diff1": 0.93499, + "nauc_ndcg_at_100_max": 0.856406, + "nauc_ndcg_at_100_std": 0.276563, + "nauc_ndcg_at_100_diff1": 0.934509, + "nauc_ndcg_at_1000_max": 0.853934, + "nauc_ndcg_at_1000_std": 0.272867, + "nauc_ndcg_at_1000_diff1": 0.933982, + "nauc_map_at_1_max": 0.80052, + "nauc_map_at_1_std": 0.215234, + "nauc_map_at_1_diff1": 0.92565, + "nauc_map_at_3_max": 0.850527, + "nauc_map_at_3_std": 0.258583, + "nauc_map_at_3_diff1": 0.933316, + "nauc_map_at_5_max": 0.848856, + "nauc_map_at_5_std": 0.261066, + "nauc_map_at_5_diff1": 0.932804, + "nauc_map_at_10_max": 0.846816, + "nauc_map_at_10_std": 0.260201, + "nauc_map_at_10_diff1": 0.932327, + "nauc_map_at_20_max": 0.846242, + "nauc_map_at_20_std": 0.258986, + "nauc_map_at_20_diff1": 0.932558, + "nauc_map_at_100_max": 0.845859, + "nauc_map_at_100_std": 0.258784, + "nauc_map_at_100_diff1": 0.932491, + "nauc_map_at_1000_max": 0.845778, + "nauc_map_at_1000_std": 0.258723, + "nauc_map_at_1000_diff1": 0.93247, + "nauc_recall_at_1_max": 0.80052, + "nauc_recall_at_1_std": 0.215234, + "nauc_recall_at_1_diff1": 0.92565, + "nauc_recall_at_3_max": 0.946114, + "nauc_recall_at_3_std": 0.353957, + "nauc_recall_at_3_diff1": 0.948772, + "nauc_recall_at_5_max": 0.952924, + "nauc_recall_at_5_std": 0.39399, + "nauc_recall_at_5_diff1": 0.948046, + "nauc_recall_at_10_max": 0.952456, + "nauc_recall_at_10_std": 0.417781, + "nauc_recall_at_10_diff1": 0.944048, + "nauc_recall_at_20_max": 0.959207, + "nauc_recall_at_20_std": 0.420818, + "nauc_recall_at_20_diff1": 0.955486, + "nauc_recall_at_100_max": 0.977402, + "nauc_recall_at_100_std": 0.571386, + "nauc_recall_at_100_diff1": 0.96275, + "nauc_recall_at_1000_max": 0.993464, + "nauc_recall_at_1000_std": 0.926003, + "nauc_recall_at_1000_diff1": 0.980391, + "nauc_precision_at_1_max": 0.80052, + "nauc_precision_at_1_std": 0.215234, + "nauc_precision_at_1_diff1": 0.92565, + "nauc_precision_at_3_max": 0.946114, + "nauc_precision_at_3_std": 0.353957, + "nauc_precision_at_3_diff1": 0.948772, + "nauc_precision_at_5_max": 0.952924, + "nauc_precision_at_5_std": 0.39399, + "nauc_precision_at_5_diff1": 0.948046, + "nauc_precision_at_10_max": 0.952456, + "nauc_precision_at_10_std": 0.417781, + "nauc_precision_at_10_diff1": 0.944048, + "nauc_precision_at_20_max": 0.959207, + "nauc_precision_at_20_std": 0.420818, + "nauc_precision_at_20_diff1": 0.955486, + "nauc_precision_at_100_max": 0.977402, + "nauc_precision_at_100_std": 0.571386, + "nauc_precision_at_100_diff1": 0.96275, + "nauc_precision_at_1000_max": 0.993464, + "nauc_precision_at_1000_std": 0.926003, + "nauc_precision_at_1000_diff1": 0.980391, + "nauc_mrr_at_1_max": 0.80052, + "nauc_mrr_at_1_std": 0.215234, + "nauc_mrr_at_1_diff1": 0.92565, + "nauc_mrr_at_3_max": 0.850527, + "nauc_mrr_at_3_std": 0.258583, + "nauc_mrr_at_3_diff1": 0.933316, + "nauc_mrr_at_5_max": 0.848856, + "nauc_mrr_at_5_std": 0.261066, + "nauc_mrr_at_5_diff1": 0.932804, + "nauc_mrr_at_10_max": 0.846816, + "nauc_mrr_at_10_std": 0.260201, + "nauc_mrr_at_10_diff1": 0.932327, + "nauc_mrr_at_20_max": 0.846242, + "nauc_mrr_at_20_std": 0.258986, + "nauc_mrr_at_20_diff1": 0.932558, + "nauc_mrr_at_100_max": 0.845859, + "nauc_mrr_at_100_std": 0.258784, + "nauc_mrr_at_100_diff1": 0.932491, + "nauc_mrr_at_1000_max": 0.845778, + "nauc_mrr_at_1000_std": 0.258723, + "nauc_mrr_at_1000_diff1": 0.93247, + "hit_rate_at_1": 0.91726, + "hit_rate_at_3": 0.95804, + "hit_rate_at_5": 0.9652, + "hit_rate_at_10": 0.97427, + "hit_rate_at_20": 0.98025, + "hit_rate_at_100": 0.98996, + "hit_rate_at_1000": 0.99786, + "main_score": 0.94799, + "hf_subset": "default", + "languages": [ + "dan-Latn" + ] + } + ] + }, + "evaluation_time": 4776.975105762482, + "kg_co2_emissions": null, + "date": 1777880190.164846 +} \ No newline at end of file diff --git a/results/Octen__Octen-Embedding-0.6B/1a00a4e837bd788f6f8d91bc43201a5e52cf8ef8/PublicNewsRetrieval.json b/results/Octen__Octen-Embedding-0.6B/1a00a4e837bd788f6f8d91bc43201a5e52cf8ef8/PublicNewsRetrieval.json new file mode 100644 index 0000000000..991e63f069 --- /dev/null +++ b/results/Octen__Octen-Embedding-0.6B/1a00a4e837bd788f6f8d91bc43201a5e52cf8ef8/PublicNewsRetrieval.json @@ -0,0 +1,322 @@ +{ + "dataset_revision": "82dad8c469646feae9631d69ba92bebfc0a586e9", + "task_name": "PublicNewsRetrieval", + "mteb_version": "2.12.37", + "scores": { + "test": [ + { + "ndcg_at_1": 0.6937, + "ndcg_at_3": 0.74489, + "ndcg_at_5": 0.76229, + "ndcg_at_10": 0.77465, + "ndcg_at_20": 0.7861, + "ndcg_at_100": 0.79657, + "ndcg_at_1000": 0.8019, + "map_at_1": 0.68527, + "map_at_3": 0.73086, + "map_at_5": 0.74089, + "map_at_10": 0.74625, + "map_at_20": 0.74962, + "map_at_100": 0.75113, + "map_at_1000": 0.75135, + "recall_at_1": 0.68527, + "recall_at_3": 0.78016, + "recall_at_5": 0.82138, + "recall_at_10": 0.85903, + "recall_at_20": 0.90265, + "recall_at_100": 0.95789, + "recall_at_1000": 0.99793, + "accuracy": 0.68527, + "precision_at_1": 0.6937, + "precision_at_3": 0.26653, + "precision_at_5": 0.16917, + "precision_at_10": 0.08861, + "precision_at_20": 0.04678, + "precision_at_100": 0.00996, + "precision_at_1000": 0.00104, + "mrr_at_1": 0.6937, + "mrr_at_3": 0.735702, + "mrr_at_5": 0.745286, + "mrr_at_10": 0.75, + "mrr_at_20": 0.75299, + "mrr_at_100": 0.754413, + "mrr_at_1000": 0.754601, + "nauc_ndcg_at_1_max": 0.517175, + "nauc_ndcg_at_1_std": -0.444717, + "nauc_ndcg_at_1_diff1": 0.835204, + "nauc_ndcg_at_3_max": 0.539892, + "nauc_ndcg_at_3_std": -0.461355, + "nauc_ndcg_at_3_diff1": 0.821472, + "nauc_ndcg_at_5_max": 0.553696, + "nauc_ndcg_at_5_std": -0.462208, + "nauc_ndcg_at_5_diff1": 0.814612, + "nauc_ndcg_at_10_max": 0.564077, + "nauc_ndcg_at_10_std": -0.456992, + "nauc_ndcg_at_10_diff1": 0.813311, + "nauc_ndcg_at_20_max": 0.565243, + "nauc_ndcg_at_20_std": -0.448366, + "nauc_ndcg_at_20_diff1": 0.811909, + "nauc_ndcg_at_100_max": 0.55891, + "nauc_ndcg_at_100_std": -0.441456, + "nauc_ndcg_at_100_diff1": 0.814238, + "nauc_ndcg_at_1000_max": 0.554355, + "nauc_ndcg_at_1000_std": -0.448036, + "nauc_ndcg_at_1000_diff1": 0.817535, + "nauc_map_at_1_max": 0.495008, + "nauc_map_at_1_std": -0.438367, + "nauc_map_at_1_diff1": 0.838408, + "nauc_map_at_3_max": 0.531325, + "nauc_map_at_3_std": -0.45557, + "nauc_map_at_3_diff1": 0.825993, + "nauc_map_at_5_max": 0.53858, + "nauc_map_at_5_std": -0.456265, + "nauc_map_at_5_diff1": 0.822721, + "nauc_map_at_10_max": 0.543131, + "nauc_map_at_10_std": -0.454164, + "nauc_map_at_10_diff1": 0.8225, + "nauc_map_at_20_max": 0.543314, + "nauc_map_at_20_std": -0.451859, + "nauc_map_at_20_diff1": 0.822089, + "nauc_map_at_100_max": 0.542547, + "nauc_map_at_100_std": -0.45072, + "nauc_map_at_100_diff1": 0.822354, + "nauc_map_at_1000_max": 0.542409, + "nauc_map_at_1000_std": -0.450938, + "nauc_map_at_1000_diff1": 0.822489, + "nauc_recall_at_1_max": 0.495008, + "nauc_recall_at_1_std": -0.438367, + "nauc_recall_at_1_diff1": 0.838408, + "nauc_recall_at_3_max": 0.55713, + "nauc_recall_at_3_std": -0.476857, + "nauc_recall_at_3_diff1": 0.806733, + "nauc_recall_at_5_max": 0.597417, + "nauc_recall_at_5_std": -0.482847, + "nauc_recall_at_5_diff1": 0.781003, + "nauc_recall_at_10_max": 0.653063, + "nauc_recall_at_10_std": -0.463659, + "nauc_recall_at_10_diff1": 0.768478, + "nauc_recall_at_20_max": 0.696476, + "nauc_recall_at_20_std": -0.402202, + "nauc_recall_at_20_diff1": 0.742438, + "nauc_recall_at_100_max": 0.7204, + "nauc_recall_at_100_std": -0.193574, + "nauc_recall_at_100_diff1": 0.707362, + "nauc_recall_at_1000_max": 0.894457, + "nauc_recall_at_1000_std": 0.040953, + "nauc_recall_at_1000_diff1": 0.791125, + "nauc_precision_at_1_max": 0.517175, + "nauc_precision_at_1_std": -0.444717, + "nauc_precision_at_1_diff1": 0.835204, + "nauc_precision_at_3_max": 0.543958, + "nauc_precision_at_3_std": -0.425177, + "nauc_precision_at_3_diff1": 0.68806, + "nauc_precision_at_5_max": 0.570044, + "nauc_precision_at_5_std": -0.384791, + "nauc_precision_at_5_diff1": 0.607404, + "nauc_precision_at_10_max": 0.586338, + "nauc_precision_at_10_std": -0.314801, + "nauc_precision_at_10_diff1": 0.519028, + "nauc_precision_at_20_max": 0.538428, + "nauc_precision_at_20_std": -0.165628, + "nauc_precision_at_20_diff1": 0.345208, + "nauc_precision_at_100_max": 0.388835, + "nauc_precision_at_100_std": 0.077089, + "nauc_precision_at_100_diff1": 0.048319, + "nauc_precision_at_1000_max": 0.071296, + "nauc_precision_at_1000_std": 0.230534, + "nauc_precision_at_1000_diff1": -0.342452, + "nauc_mrr_at_1_max": 0.517175, + "nauc_mrr_at_1_std": -0.444717, + "nauc_mrr_at_1_diff1": 0.835204, + "nauc_mrr_at_3_max": 0.538884, + "nauc_mrr_at_3_std": -0.458965, + "nauc_mrr_at_3_diff1": 0.823801, + "nauc_mrr_at_5_max": 0.54729, + "nauc_mrr_at_5_std": -0.458217, + "nauc_mrr_at_5_diff1": 0.820284, + "nauc_mrr_at_10_max": 0.549311, + "nauc_mrr_at_10_std": -0.457453, + "nauc_mrr_at_10_diff1": 0.819994, + "nauc_mrr_at_20_max": 0.548203, + "nauc_mrr_at_20_std": -0.455995, + "nauc_mrr_at_20_diff1": 0.819906, + "nauc_mrr_at_100_max": 0.547386, + "nauc_mrr_at_100_std": -0.455244, + "nauc_mrr_at_100_diff1": 0.820262, + "nauc_mrr_at_1000_max": 0.547253, + "nauc_mrr_at_1000_std": -0.455356, + "nauc_mrr_at_1000_diff1": 0.820344, + "hit_rate_at_1": 0.6937, + "hit_rate_at_3": 0.78686, + "hit_rate_at_5": 0.82842, + "hit_rate_at_10": 0.86394, + "hit_rate_at_20": 0.9055, + "hit_rate_at_100": 0.96113, + "hit_rate_at_1000": 0.99866, + "main_score": 0.77465, + "hf_subset": "pl", + "languages": [ + "pol-Latn" + ] + }, + { + "ndcg_at_1": 0.63924, + "ndcg_at_3": 0.71897, + "ndcg_at_5": 0.74047, + "ndcg_at_10": 0.75961, + "ndcg_at_20": 0.76954, + "ndcg_at_100": 0.78155, + "ndcg_at_1000": 0.7841, + "map_at_1": 0.62136, + "map_at_3": 0.69614, + "map_at_5": 0.7088, + "map_at_10": 0.71749, + "map_at_20": 0.7204, + "map_at_100": 0.72224, + "map_at_1000": 0.72237, + "recall_at_1": 0.62136, + "recall_at_3": 0.77549, + "recall_at_5": 0.82577, + "recall_at_10": 0.88235, + "recall_at_20": 0.92025, + "recall_at_100": 0.9818, + "recall_at_1000": 0.99976, + "accuracy": 0.62136, + "precision_at_1": 0.63924, + "precision_at_3": 0.27139, + "precision_at_5": 0.17409, + "precision_at_10": 0.09357, + "precision_at_20": 0.04889, + "precision_at_100": 0.01048, + "precision_at_1000": 0.00107, + "mrr_at_1": 0.639243, + "mrr_at_3": 0.705516, + "mrr_at_5": 0.716958, + "mrr_at_10": 0.724119, + "mrr_at_20": 0.726771, + "mrr_at_100": 0.728368, + "mrr_at_1000": 0.728457, + "nauc_ndcg_at_1_max": 0.364588, + "nauc_ndcg_at_1_std": -0.438741, + "nauc_ndcg_at_1_diff1": 0.809784, + "nauc_ndcg_at_3_max": 0.373421, + "nauc_ndcg_at_3_std": -0.506644, + "nauc_ndcg_at_3_diff1": 0.77857, + "nauc_ndcg_at_5_max": 0.374241, + "nauc_ndcg_at_5_std": -0.523693, + "nauc_ndcg_at_5_diff1": 0.776555, + "nauc_ndcg_at_10_max": 0.378633, + "nauc_ndcg_at_10_std": -0.526684, + "nauc_ndcg_at_10_diff1": 0.77776, + "nauc_ndcg_at_20_max": 0.374711, + "nauc_ndcg_at_20_std": -0.52433, + "nauc_ndcg_at_20_diff1": 0.777583, + "nauc_ndcg_at_100_max": 0.373202, + "nauc_ndcg_at_100_std": -0.511317, + "nauc_ndcg_at_100_diff1": 0.782623, + "nauc_ndcg_at_1000_max": 0.373436, + "nauc_ndcg_at_1000_std": -0.505598, + "nauc_ndcg_at_1000_diff1": 0.783664, + "nauc_map_at_1_max": 0.337528, + "nauc_map_at_1_std": -0.425148, + "nauc_map_at_1_diff1": 0.806375, + "nauc_map_at_3_max": 0.364889, + "nauc_map_at_3_std": -0.490309, + "nauc_map_at_3_diff1": 0.785923, + "nauc_map_at_5_max": 0.365901, + "nauc_map_at_5_std": -0.499193, + "nauc_map_at_5_diff1": 0.785174, + "nauc_map_at_10_max": 0.368463, + "nauc_map_at_10_std": -0.499705, + "nauc_map_at_10_diff1": 0.786061, + "nauc_map_at_20_max": 0.367527, + "nauc_map_at_20_std": -0.498859, + "nauc_map_at_20_diff1": 0.786134, + "nauc_map_at_100_max": 0.367372, + "nauc_map_at_100_std": -0.497143, + "nauc_map_at_100_diff1": 0.786822, + "nauc_map_at_1000_max": 0.367384, + "nauc_map_at_1000_std": -0.49685, + "nauc_map_at_1000_diff1": 0.786859, + "nauc_recall_at_1_max": 0.337528, + "nauc_recall_at_1_std": -0.425148, + "nauc_recall_at_1_diff1": 0.806375, + "nauc_recall_at_3_max": 0.386727, + "nauc_recall_at_3_std": -0.56621, + "nauc_recall_at_3_diff1": 0.746664, + "nauc_recall_at_5_max": 0.386983, + "nauc_recall_at_5_std": -0.634778, + "nauc_recall_at_5_diff1": 0.728791, + "nauc_recall_at_10_max": 0.414807, + "nauc_recall_at_10_std": -0.701271, + "nauc_recall_at_10_diff1": 0.72022, + "nauc_recall_at_20_max": 0.390688, + "nauc_recall_at_20_std": -0.755183, + "nauc_recall_at_20_diff1": 0.691537, + "nauc_recall_at_100_max": 0.360666, + "nauc_recall_at_100_std": -0.889334, + "nauc_recall_at_100_diff1": 0.696486, + "nauc_recall_at_1000_max": 0.12258, + "nauc_recall_at_1000_std": -0.564004, + "nauc_recall_at_1000_diff1": 0.722188, + "nauc_precision_at_1_max": 0.364588, + "nauc_precision_at_1_std": -0.438741, + "nauc_precision_at_1_diff1": 0.809784, + "nauc_precision_at_3_max": 0.368537, + "nauc_precision_at_3_std": -0.472222, + "nauc_precision_at_3_diff1": 0.562226, + "nauc_precision_at_5_max": 0.35132, + "nauc_precision_at_5_std": -0.469572, + "nauc_precision_at_5_diff1": 0.46875, + "nauc_precision_at_10_max": 0.313688, + "nauc_precision_at_10_std": -0.384732, + "nauc_precision_at_10_diff1": 0.309214, + "nauc_precision_at_20_max": 0.261074, + "nauc_precision_at_20_std": -0.299194, + "nauc_precision_at_20_diff1": 0.173412, + "nauc_precision_at_100_max": 0.132358, + "nauc_precision_at_100_std": 0.038952, + "nauc_precision_at_100_diff1": -0.158319, + "nauc_precision_at_1000_max": 0.074634, + "nauc_precision_at_1000_std": 0.213913, + "nauc_precision_at_1000_diff1": -0.29019, + "nauc_mrr_at_1_max": 0.364588, + "nauc_mrr_at_1_std": -0.438741, + "nauc_mrr_at_1_diff1": 0.809784, + "nauc_mrr_at_3_max": 0.377172, + "nauc_mrr_at_3_std": -0.486255, + "nauc_mrr_at_3_diff1": 0.787979, + "nauc_mrr_at_5_max": 0.377175, + "nauc_mrr_at_5_std": -0.492444, + "nauc_mrr_at_5_diff1": 0.787529, + "nauc_mrr_at_10_max": 0.376959, + "nauc_mrr_at_10_std": -0.493204, + "nauc_mrr_at_10_diff1": 0.787969, + "nauc_mrr_at_20_max": 0.375825, + "nauc_mrr_at_20_std": -0.492832, + "nauc_mrr_at_20_diff1": 0.788075, + "nauc_mrr_at_100_max": 0.375576, + "nauc_mrr_at_100_std": -0.491551, + "nauc_mrr_at_100_diff1": 0.788642, + "nauc_mrr_at_1000_max": 0.375574, + "nauc_mrr_at_1000_std": -0.491401, + "nauc_mrr_at_1000_diff1": 0.788669, + "hit_rate_at_1": 0.63924, + "hit_rate_at_3": 0.78723, + "hit_rate_at_5": 0.83641, + "hit_rate_at_10": 0.88889, + "hit_rate_at_20": 0.92624, + "hit_rate_at_100": 0.98534, + "hit_rate_at_1000": 1.0, + "main_score": 0.75961, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ] + } + ] + }, + "evaluation_time": 379.3539810180664, + "kg_co2_emissions": null, + "date": 1777965773.762743 +} \ No newline at end of file diff --git a/results/Octen__Octen-Embedding-0.6B/1a00a4e837bd788f6f8d91bc43201a5e52cf8ef8/model_meta.json b/results/Octen__Octen-Embedding-0.6B/1a00a4e837bd788f6f8d91bc43201a5e52cf8ef8/model_meta.json index a66680bb90..13bbb39e0c 100644 --- a/results/Octen__Octen-Embedding-0.6B/1a00a4e837bd788f6f8d91bc43201a5e52cf8ef8/model_meta.json +++ b/results/Octen__Octen-Embedding-0.6B/1a00a4e837bd788f6f8d91bc43201a5e52cf8ef8/model_meta.json @@ -1 +1,165 @@ -{"loader_kwargs": {"instruction_template": "", "apply_instruction_to_passages": true, "prompts_dict": {"AILACasedocs": "Given a legal case scenario, retrieve the most relevant case documents", "AILAStatutes": "Given a legal scenario, retrieve the most relevant statute documents", "LegalQuAD": "Given a legal question, retrieve relevant legal documents that answer the question", "LegalSummarization": "Given a query, retrieve relevant legal documents for summarization", "AppsRetrieval": "Given a query about mobile applications, retrieve relevant app information", "HumanEvalRetrieval": "Given a code problem description, retrieve relevant code examples", "MBPPRetrieval": "Given a programming problem description, retrieve relevant code solutions", "DS1000Retrieval": "Given a data science problem, retrieve relevant code snippets", "FreshStackRetrieval": "Given a programming question, retrieve relevant Stack Overflow posts", "FinQARetrieval": "Given a financial question, retrieve relevant financial documents", "FinanceBenchRetrieval": "Given a financial query, retrieve relevant financial information", "HC3FinanceRetrieval": "Given a finance-related query, retrieve relevant documents", "CUREv1": "Given a medical query, retrieve relevant clinical documents", "ChatDoctorRetrieval": "Given a medical question, retrieve relevant medical information", "WikiSQLRetrieval": "Given a natural language query, retrieve relevant SQL examples", "MIRACLRetrievalHardNegatives": "Given a question, retrieve Wikipedia passages that answer the question", "Code1Retrieval": "Given a code problem description, retrieve relevant code examples", "JapaneseCode1Retrieval": "Given a code problem description, retrieve relevant code examples", "EnglishFinance1Retrieval": "Given a financial query, retrieve relevant financial documents", "EnglishFinance2Retrieval": "Given a financial query, retrieve relevant financial documents", "EnglishFinance3Retrieval": "Given a financial query, retrieve relevant financial documents", "EnglishFinance4Retrieval": "Given a financial query, retrieve relevant financial documents", "EnglishHealthcare1Retrieval": "Given a medical question, retrieve relevant medical information", "GermanHealthcare1Retrieval": "Given a medical question, retrieve relevant medical information", "FrenchLegal1Retrieval": "Given a legal query, retrieve relevant legal documents", "GermanLegal1Retrieval": "Given a legal query, retrieve relevant legal documents", "JapaneseLegal1Retrieval": "Given a legal query, retrieve relevant legal documents", "French1Retrieval": "Given a query, retrieve relevant passages", "German1Retrieval": "Given a query, retrieve relevant passages"}, "max_seq_length": 18480, "model_kwargs": {"torch_dtype": "bfloat16"}}, "name": "Octen/Octen-Embedding-0.6B", "revision": "1a00a4e837bd788f6f8d91bc43201a5e52cf8ef8", "release_date": "2026-01-10", "languages": ["afr-Latn", "ara-Arab", "aze-Latn", "bel-Cyrl", "bul-Cyrl", "ben-Beng", "cat-Latn", "ceb-Latn", "ces-Latn", "cym-Latn", "dan-Latn", "deu-Latn", "ell-Grek", "eng-Latn", "spa-Latn", "est-Latn", "eus-Latn", "fas-Arab", "fin-Latn", "fra-Latn", "glg-Latn", "guj-Gujr", "heb-Hebr", "hin-Deva", "hrv-Latn", "hat-Latn", "hun-Latn", "hye-Armn", "ind-Latn", "isl-Latn", "ita-Latn", "jpn-Jpan", "jav-Latn", "kat-Geor", "kaz-Cyrl", "khm-Khmr", "kan-Knda", "kor-Hang", "kir-Cyrl", "lao-Laoo", "lit-Latn", "lav-Latn", "mkd-Cyrl", "mal-Mlym", "mon-Cyrl", "mar-Deva", "msa-Latn", "mya-Mymr", "nep-Deva", "nld-Latn", "nor-Latn", "nob-Latn", "nno-Latn", "pan-Guru", "pol-Latn", "por-Latn", "que-Latn", "ron-Latn", "rus-Cyrl", "sin-Sinh", "slk-Latn", "slv-Latn", "swa-Latn", "tam-Taml", "tel-Telu", "tha-Thai", "tgl-Latn", "tur-Latn", "ukr-Cyrl", "urd-Arab", "vie-Latn", "yor-Latn", "zho-Hans"], "n_parameters": 595776512, "memory_usage_mb": 1136.0, "max_tokens": 32768.0, "embed_dim": 1024, "license": "apache-2.0", "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["Sentence Transformers", "PyTorch", "safetensors"], "reference": "https://huggingface.co/Octen/Octen-Embedding-0.6B", "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": ["MIRACLRetrieval", "CodeSearchNet", "NQ", "MMarcoReranking", "CMedQAv2-reranking", "MrTidyRetrieval", "DuRetrieval", "FEVER", "HotpotQA", "MSMARCO", "T2Retrieval"], "adapted_from": "Qwen/Qwen3-Embedding-0.6B", "superseded_by": null, "modalities": ["text"], "model_type": ["dense"], "citation": "@misc{octen-embedding-2025,\n title={Octen-Embedding-8B: A Fine-tuned Multilingual Text Embedding Model},\n author={Octen Team},\n year={2025},\n url={https://huggingface.co/Octen/Octen-Embedding-8B}\n}", "contacts": null, "loader": "InstructSentenceTransformerModel", "is_cross_encoder": false} +{ + "loader_kwargs": { + "instruction_template": "", + "apply_instruction_to_passages": true, + "prompts_dict": { + "AILACasedocs": "Given a legal case scenario, retrieve the most relevant case documents", + "AILAStatutes": "Given a legal scenario, retrieve the most relevant statute documents", + "LegalQuAD": "Given a legal question, retrieve relevant legal documents that answer the question", + "LegalSummarization": "Given a query, retrieve relevant legal documents for summarization", + "AppsRetrieval": "Given a query about mobile applications, retrieve relevant app information", + "HumanEvalRetrieval": "Given a code problem description, retrieve relevant code examples", + "MBPPRetrieval": "Given a programming problem description, retrieve relevant code solutions", + "DS1000Retrieval": "Given a data science problem, retrieve relevant code snippets", + "FreshStackRetrieval": "Given a programming question, retrieve relevant Stack Overflow posts", + "FinQARetrieval": "Given a financial question, retrieve relevant financial documents", + "FinanceBenchRetrieval": "Given a financial query, retrieve relevant financial information", + "HC3FinanceRetrieval": "Given a finance-related query, retrieve relevant documents", + "CUREv1": "Given a medical query, retrieve relevant clinical documents", + "ChatDoctorRetrieval": "Given a medical question, retrieve relevant medical information", + "WikiSQLRetrieval": "Given a natural language query, retrieve relevant SQL examples", + "MIRACLRetrievalHardNegatives": "Given a question, retrieve Wikipedia passages that answer the question", + "Code1Retrieval": "Given a code problem description, retrieve relevant code examples", + "JapaneseCode1Retrieval": "Given a code problem description, retrieve relevant code examples", + "EnglishFinance1Retrieval": "Given a financial query, retrieve relevant financial documents", + "EnglishFinance2Retrieval": "Given a financial query, retrieve relevant financial documents", + "EnglishFinance3Retrieval": "Given a financial query, retrieve relevant financial documents", + "EnglishFinance4Retrieval": "Given a financial query, retrieve relevant financial documents", + "EnglishHealthcare1Retrieval": "Given a medical question, retrieve relevant medical information", + "GermanHealthcare1Retrieval": "Given a medical question, retrieve relevant medical information", + "FrenchLegal1Retrieval": "Given a legal query, retrieve relevant legal documents", + "GermanLegal1Retrieval": "Given a legal query, retrieve relevant legal documents", + "JapaneseLegal1Retrieval": "Given a legal query, retrieve relevant legal documents", + "French1Retrieval": "Given a query, retrieve relevant passages", + "German1Retrieval": "Given a query, retrieve relevant passages" + }, + "max_seq_length": 18480, + "model_kwargs": { + "torch_dtype": "bfloat16" + } + }, + "name": "Octen/Octen-Embedding-0.6B", + "revision": "1a00a4e837bd788f6f8d91bc43201a5e52cf8ef8", + "release_date": "2026-01-10", + "languages": [ + "afr-Latn", + "ara-Arab", + "aze-Latn", + "bel-Cyrl", + "bul-Cyrl", + "ben-Beng", + "cat-Latn", + "ceb-Latn", + "ces-Latn", + "cym-Latn", + "dan-Latn", + "deu-Latn", + "ell-Grek", + "eng-Latn", + "spa-Latn", + "est-Latn", + "eus-Latn", + "fas-Arab", + "fin-Latn", + "fra-Latn", + "glg-Latn", + "guj-Gujr", + "heb-Hebr", + "hin-Deva", + "hrv-Latn", + "hat-Latn", + "hun-Latn", + "hye-Armn", + "ind-Latn", + "isl-Latn", + "ita-Latn", + "jpn-Jpan", + "jav-Latn", + "kat-Geor", + "kaz-Cyrl", + "khm-Khmr", + "kan-Knda", + "kor-Hang", + "kir-Cyrl", + "lao-Laoo", + "lit-Latn", + "lav-Latn", + "mkd-Cyrl", + "mal-Mlym", + "mon-Cyrl", + "mar-Deva", + "msa-Latn", + "mya-Mymr", + "nep-Deva", + "nld-Latn", + "nor-Latn", + "nob-Latn", + "nno-Latn", + "pan-Guru", + "pol-Latn", + "por-Latn", + "que-Latn", + "ron-Latn", + "rus-Cyrl", + "sin-Sinh", + "slk-Latn", + "slv-Latn", + "swa-Latn", + "tam-Taml", + "tel-Telu", + "tha-Thai", + "tgl-Latn", + "tur-Latn", + "ukr-Cyrl", + "urd-Arab", + "vie-Latn", + "yor-Latn", + "zho-Hans" + ], + "n_parameters": 595776512, + "n_active_parameters_override": null, + "n_embedding_parameters": 155309056, + "memory_usage_mb": 1136.0, + "max_tokens": 32768.0, + "embed_dim": 1024, + "license": "apache-2.0", + "open_weights": true, + "public_training_code": null, + "public_training_data": null, + "framework": [ + "Sentence Transformers", + "PyTorch", + "safetensors" + ], + "reference": "https://huggingface.co/Octen/Octen-Embedding-0.6B", + "similarity_fn_name": "cosine", + "use_instructions": true, + "training_datasets": [ + "MMarcoReranking", + "T2Retrieval", + "FEVER", + "MSMARCO", + "NQ", + "HotpotQA", + "CodeSearchNet", + "MIRACLRetrieval", + "CMedQAv2-reranking", + "DuRetrieval", + "MrTidyRetrieval" + ], + "adapted_from": "Qwen/Qwen3-Embedding-0.6B", + "superseded_by": null, + "modalities": [ + "text" + ], + "model_type": [ + "dense" + ], + "citation": "@misc{octen2025rteb,\n title={Octen Series: Optimizing Embedding Models to #1 on RTEB Leaderboard},\n author={Octen Team},\n year={2025},\n url={https://octen-team.github.io/octen_blog/posts/octen-rteb-first-place/}\n}", + "contacts": null, + "experiment_kwargs": null, + "output_dtypes": "bfloat16", + "extra_requirements_groups": null, + "loader": "InstructSentenceTransformerModel", + "is_cross_encoder": false +} \ No newline at end of file diff --git a/results/Qwen__Qwen3-Embedding-0.6B/b22da495047858cce924d27d76261e96be6febc0/ElasticKBRetrieval.json b/results/Qwen__Qwen3-Embedding-0.6B/b22da495047858cce924d27d76261e96be6febc0/ElasticKBRetrieval.json new file mode 100644 index 0000000000..ac05226d68 --- /dev/null +++ b/results/Qwen__Qwen3-Embedding-0.6B/b22da495047858cce924d27d76261e96be6febc0/ElasticKBRetrieval.json @@ -0,0 +1,324 @@ +{ + "dataset_revision": "21bdf1e024bf7c9f46720017559ce2f8c6116507", + "task_name": "ElasticKBRetrieval", + "mteb_version": "2.12.38", + "scores": { + "synthetic_test": [ + { + "ndcg_at_1": 0.71809, + "ndcg_at_3": 0.55518, + "ndcg_at_5": 0.52066, + "ndcg_at_10": 0.49854, + "ndcg_at_20": 0.49184, + "ndcg_at_100": 0.53928, + "ndcg_at_1000": 0.6056, + "map_at_1": 0.23301, + "map_at_3": 0.28956, + "map_at_5": 0.30388, + "map_at_10": 0.32158, + "map_at_20": 0.33658, + "map_at_100": 0.36154, + "map_at_1000": 0.37199, + "recall_at_1": 0.23301, + "recall_at_3": 0.31412, + "recall_at_5": 0.34413, + "recall_at_10": 0.39936, + "recall_at_20": 0.45369, + "recall_at_100": 0.62476, + "recall_at_1000": 0.83815, + "accuracy": 0.23301, + "precision_at_1": 0.71809, + "precision_at_3": 0.40426, + "precision_at_5": 0.30957, + "precision_at_10": 0.21649, + "precision_at_20": 0.1484, + "precision_at_100": 0.06027, + "precision_at_1000": 0.01176, + "mrr_at_1": 0.718085, + "mrr_at_3": 0.782801, + "mrr_at_5": 0.795301, + "mrr_at_10": 0.801433, + "mrr_at_20": 0.802463, + "mrr_at_100": 0.803132, + "mrr_at_1000": 0.803132, + "nauc_ndcg_at_1_max": 0.187853, + "nauc_ndcg_at_1_std": -0.267116, + "nauc_ndcg_at_1_diff1": 0.642781, + "nauc_ndcg_at_3_max": 0.085363, + "nauc_ndcg_at_3_std": -0.137304, + "nauc_ndcg_at_3_diff1": 0.322082, + "nauc_ndcg_at_5_max": 0.108737, + "nauc_ndcg_at_5_std": -0.091668, + "nauc_ndcg_at_5_diff1": 0.330598, + "nauc_ndcg_at_10_max": 0.135758, + "nauc_ndcg_at_10_std": -0.074638, + "nauc_ndcg_at_10_diff1": 0.375076, + "nauc_ndcg_at_20_max": 0.148662, + "nauc_ndcg_at_20_std": -0.048643, + "nauc_ndcg_at_20_diff1": 0.38935, + "nauc_ndcg_at_100_max": 0.223693, + "nauc_ndcg_at_100_std": 0.044416, + "nauc_ndcg_at_100_diff1": 0.424773, + "nauc_ndcg_at_1000_max": 0.202642, + "nauc_ndcg_at_1000_std": 0.053972, + "nauc_ndcg_at_1000_diff1": 0.403753, + "nauc_map_at_1_max": 0.087736, + "nauc_map_at_1_std": -0.178063, + "nauc_map_at_1_diff1": 0.514349, + "nauc_map_at_3_max": 0.1089, + "nauc_map_at_3_std": -0.143897, + "nauc_map_at_3_diff1": 0.459655, + "nauc_map_at_5_max": 0.121641, + "nauc_map_at_5_std": -0.133321, + "nauc_map_at_5_diff1": 0.453868, + "nauc_map_at_10_max": 0.131203, + "nauc_map_at_10_std": -0.117017, + "nauc_map_at_10_diff1": 0.447007, + "nauc_map_at_20_max": 0.146503, + "nauc_map_at_20_std": -0.083904, + "nauc_map_at_20_diff1": 0.435876, + "nauc_map_at_100_max": 0.171052, + "nauc_map_at_100_std": -0.036517, + "nauc_map_at_100_diff1": 0.430385, + "nauc_map_at_1000_max": 0.166707, + "nauc_map_at_1000_std": -0.029893, + "nauc_map_at_1000_diff1": 0.427414, + "nauc_recall_at_1_max": 0.087736, + "nauc_recall_at_1_std": -0.178063, + "nauc_recall_at_1_diff1": 0.514349, + "nauc_recall_at_3_max": 0.110194, + "nauc_recall_at_3_std": -0.109257, + "nauc_recall_at_3_diff1": 0.392704, + "nauc_recall_at_5_max": 0.140473, + "nauc_recall_at_5_std": -0.079242, + "nauc_recall_at_5_diff1": 0.372919, + "nauc_recall_at_10_max": 0.146332, + "nauc_recall_at_10_std": -0.060596, + "nauc_recall_at_10_diff1": 0.33918, + "nauc_recall_at_20_max": 0.160767, + "nauc_recall_at_20_std": 0.016718, + "nauc_recall_at_20_diff1": 0.300335, + "nauc_recall_at_100_max": 0.229089, + "nauc_recall_at_100_std": 0.198389, + "nauc_recall_at_100_diff1": 0.304321, + "nauc_recall_at_1000_max": 0.191633, + "nauc_recall_at_1000_std": 0.246494, + "nauc_recall_at_1000_diff1": 0.234038, + "nauc_precision_at_1_max": 0.187853, + "nauc_precision_at_1_std": -0.267116, + "nauc_precision_at_1_diff1": 0.642781, + "nauc_precision_at_3_max": 0.034816, + "nauc_precision_at_3_std": -0.051568, + "nauc_precision_at_3_diff1": -0.00724, + "nauc_precision_at_5_max": 0.039661, + "nauc_precision_at_5_std": 0.085534, + "nauc_precision_at_5_diff1": -0.144806, + "nauc_precision_at_10_max": 0.041057, + "nauc_precision_at_10_std": 0.22112, + "nauc_precision_at_10_diff1": -0.273401, + "nauc_precision_at_20_max": 0.031501, + "nauc_precision_at_20_std": 0.293258, + "nauc_precision_at_20_diff1": -0.310288, + "nauc_precision_at_100_max": -0.044119, + "nauc_precision_at_100_std": 0.299786, + "nauc_precision_at_100_diff1": -0.294831, + "nauc_precision_at_1000_max": -0.191569, + "nauc_precision_at_1000_std": 0.117812, + "nauc_precision_at_1000_diff1": -0.309326, + "nauc_mrr_at_1_max": 0.187853, + "nauc_mrr_at_1_std": -0.267116, + "nauc_mrr_at_1_diff1": 0.642781, + "nauc_mrr_at_3_max": 0.197475, + "nauc_mrr_at_3_std": -0.239147, + "nauc_mrr_at_3_diff1": 0.614714, + "nauc_mrr_at_5_max": 0.236659, + "nauc_mrr_at_5_std": -0.217964, + "nauc_mrr_at_5_diff1": 0.610839, + "nauc_mrr_at_10_max": 0.223098, + "nauc_mrr_at_10_std": -0.232776, + "nauc_mrr_at_10_diff1": 0.617536, + "nauc_mrr_at_20_max": 0.221383, + "nauc_mrr_at_20_std": -0.228135, + "nauc_mrr_at_20_diff1": 0.615735, + "nauc_mrr_at_100_max": 0.22019, + "nauc_mrr_at_100_std": -0.23017, + "nauc_mrr_at_100_diff1": 0.616755, + "nauc_mrr_at_1000_max": 0.22019, + "nauc_mrr_at_1000_std": -0.23017, + "nauc_mrr_at_1000_diff1": 0.616755, + "hit_rate_at_1": 0.71809, + "hit_rate_at_3": 0.86702, + "hit_rate_at_5": 0.92021, + "hit_rate_at_10": 0.96277, + "hit_rate_at_20": 0.97872, + "hit_rate_at_100": 1.0, + "hit_rate_at_1000": 1.0, + "main_score": 0.49854, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ], + "real_chat_test": [ + { + "ndcg_at_1": 0.67672, + "ndcg_at_3": 0.61053, + "ndcg_at_5": 0.56659, + "ndcg_at_10": 0.52188, + "ndcg_at_20": 0.48276, + "ndcg_at_100": 0.44501, + "ndcg_at_1000": 0.53395, + "map_at_1": 0.05508, + "map_at_3": 0.0934, + "map_at_5": 0.10938, + "map_at_10": 0.13059, + "map_at_20": 0.14849, + "map_at_100": 0.18925, + "map_at_1000": 0.23774, + "recall_at_1": 0.05508, + "recall_at_3": 0.10469, + "recall_at_5": 0.13139, + "recall_at_10": 0.17102, + "recall_at_20": 0.21884, + "recall_at_100": 0.37439, + "recall_at_1000": 0.6937, + "accuracy": 0.05508, + "precision_at_1": 0.67672, + "precision_at_3": 0.57615, + "precision_at_5": 0.50776, + "precision_at_10": 0.42802, + "precision_at_20": 0.34397, + "precision_at_100": 0.2056, + "precision_at_1000": 0.07526, + "mrr_at_1": 0.676724, + "mrr_at_3": 0.779454, + "mrr_at_5": 0.787428, + "mrr_at_10": 0.789198, + "mrr_at_20": 0.790373, + "mrr_at_100": 0.790767, + "mrr_at_1000": 0.790868, + "nauc_ndcg_at_1_max": 0.280438, + "nauc_ndcg_at_1_std": 0.198581, + "nauc_ndcg_at_1_diff1": 0.311347, + "nauc_ndcg_at_3_max": 0.311356, + "nauc_ndcg_at_3_std": 0.243983, + "nauc_ndcg_at_3_diff1": 0.124704, + "nauc_ndcg_at_5_max": 0.278128, + "nauc_ndcg_at_5_std": 0.247822, + "nauc_ndcg_at_5_diff1": 0.097126, + "nauc_ndcg_at_10_max": 0.293951, + "nauc_ndcg_at_10_std": 0.261932, + "nauc_ndcg_at_10_diff1": 0.073391, + "nauc_ndcg_at_20_max": 0.306722, + "nauc_ndcg_at_20_std": 0.285095, + "nauc_ndcg_at_20_diff1": 0.047775, + "nauc_ndcg_at_100_max": 0.341679, + "nauc_ndcg_at_100_std": 0.274877, + "nauc_ndcg_at_100_diff1": 0.09235, + "nauc_ndcg_at_1000_max": 0.358166, + "nauc_ndcg_at_1000_std": 0.386748, + "nauc_ndcg_at_1000_diff1": 0.150699, + "nauc_map_at_1_max": 0.368263, + "nauc_map_at_1_std": 0.054885, + "nauc_map_at_1_diff1": 0.498711, + "nauc_map_at_3_max": 0.330468, + "nauc_map_at_3_std": 0.032981, + "nauc_map_at_3_diff1": 0.424211, + "nauc_map_at_5_max": 0.304808, + "nauc_map_at_5_std": 0.038753, + "nauc_map_at_5_diff1": 0.379695, + "nauc_map_at_10_max": 0.300031, + "nauc_map_at_10_std": 0.033222, + "nauc_map_at_10_diff1": 0.355045, + "nauc_map_at_20_max": 0.307276, + "nauc_map_at_20_std": 0.059203, + "nauc_map_at_20_diff1": 0.324878, + "nauc_map_at_100_max": 0.338488, + "nauc_map_at_100_std": 0.157547, + "nauc_map_at_100_diff1": 0.293209, + "nauc_map_at_1000_max": 0.364997, + "nauc_map_at_1000_std": 0.270127, + "nauc_map_at_1000_diff1": 0.22259, + "nauc_recall_at_1_max": 0.368263, + "nauc_recall_at_1_std": 0.054885, + "nauc_recall_at_1_diff1": 0.498711, + "nauc_recall_at_3_max": 0.306249, + "nauc_recall_at_3_std": 0.014888, + "nauc_recall_at_3_diff1": 0.412626, + "nauc_recall_at_5_max": 0.252281, + "nauc_recall_at_5_std": 0.010017, + "nauc_recall_at_5_diff1": 0.329267, + "nauc_recall_at_10_max": 0.229681, + "nauc_recall_at_10_std": -0.01143, + "nauc_recall_at_10_diff1": 0.290172, + "nauc_recall_at_20_max": 0.209469, + "nauc_recall_at_20_std": 0.013013, + "nauc_recall_at_20_diff1": 0.201862, + "nauc_recall_at_100_max": 0.213425, + "nauc_recall_at_100_std": 0.112315, + "nauc_recall_at_100_diff1": 0.157134, + "nauc_recall_at_1000_max": 0.213789, + "nauc_recall_at_1000_std": 0.253648, + "nauc_recall_at_1000_diff1": 0.076269, + "nauc_precision_at_1_max": 0.280438, + "nauc_precision_at_1_std": 0.198581, + "nauc_precision_at_1_diff1": 0.311347, + "nauc_precision_at_3_max": 0.243828, + "nauc_precision_at_3_std": 0.241613, + "nauc_precision_at_3_diff1": -0.017871, + "nauc_precision_at_5_max": 0.164507, + "nauc_precision_at_5_std": 0.22814, + "nauc_precision_at_5_diff1": -0.081696, + "nauc_precision_at_10_max": 0.143325, + "nauc_precision_at_10_std": 0.230706, + "nauc_precision_at_10_diff1": -0.171073, + "nauc_precision_at_20_max": 0.105419, + "nauc_precision_at_20_std": 0.281797, + "nauc_precision_at_20_diff1": -0.214366, + "nauc_precision_at_100_max": 0.038348, + "nauc_precision_at_100_std": 0.284005, + "nauc_precision_at_100_diff1": -0.260934, + "nauc_precision_at_1000_max": -0.034039, + "nauc_precision_at_1000_std": 0.116305, + "nauc_precision_at_1000_diff1": -0.271272, + "nauc_mrr_at_1_max": 0.280438, + "nauc_mrr_at_1_std": 0.198581, + "nauc_mrr_at_1_diff1": 0.311347, + "nauc_mrr_at_3_max": 0.307229, + "nauc_mrr_at_3_std": 0.24085, + "nauc_mrr_at_3_diff1": 0.304152, + "nauc_mrr_at_5_max": 0.303526, + "nauc_mrr_at_5_std": 0.235514, + "nauc_mrr_at_5_diff1": 0.314316, + "nauc_mrr_at_10_max": 0.306341, + "nauc_mrr_at_10_std": 0.233077, + "nauc_mrr_at_10_diff1": 0.316616, + "nauc_mrr_at_20_max": 0.305453, + "nauc_mrr_at_20_std": 0.230533, + "nauc_mrr_at_20_diff1": 0.317172, + "nauc_mrr_at_100_max": 0.306915, + "nauc_mrr_at_100_std": 0.231619, + "nauc_mrr_at_100_diff1": 0.31737, + "nauc_mrr_at_1000_max": 0.306876, + "nauc_mrr_at_1000_std": 0.231671, + "nauc_mrr_at_1000_diff1": 0.317113, + "hit_rate_at_1": 0.67672, + "hit_rate_at_3": 0.90517, + "hit_rate_at_5": 0.93966, + "hit_rate_at_10": 0.95259, + "hit_rate_at_20": 0.96983, + "hit_rate_at_100": 0.98276, + "hit_rate_at_1000": 1.0, + "main_score": 0.52188, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 1686.5352165699005, + "kg_co2_emissions": null, + "date": 1777930004.26419 +} \ No newline at end of file diff --git a/results/Qwen__Qwen3-Embedding-0.6B/b22da495047858cce924d27d76261e96be6febc0/GlobalNewsRetrieval.json b/results/Qwen__Qwen3-Embedding-0.6B/b22da495047858cce924d27d76261e96be6febc0/GlobalNewsRetrieval.json new file mode 100644 index 0000000000..210e131629 --- /dev/null +++ b/results/Qwen__Qwen3-Embedding-0.6B/b22da495047858cce924d27d76261e96be6febc0/GlobalNewsRetrieval.json @@ -0,0 +1,1562 @@ +{ + "dataset_revision": "234dcd32922d32c78e08bfe506a2b90a5401cf7c", + "task_name": "GlobalNewsRetrieval", + "mteb_version": "2.12.38", + "scores": { + "test": [ + { + "ndcg_at_1": 0.63784, + "ndcg_at_3": 0.71708, + "ndcg_at_5": 0.74125, + "ndcg_at_10": 0.76211, + "ndcg_at_20": 0.77467, + "ndcg_at_100": 0.78459, + "ndcg_at_1000": 0.78626, + "map_at_1": 0.60673, + "map_at_3": 0.68752, + "map_at_5": 0.70313, + "map_at_10": 0.71283, + "map_at_20": 0.71699, + "map_at_100": 0.71859, + "map_at_1000": 0.71869, + "recall_at_1": 0.60673, + "recall_at_3": 0.7784, + "recall_at_5": 0.83229, + "recall_at_10": 0.89307, + "recall_at_20": 0.93901, + "recall_at_100": 0.98851, + "recall_at_1000": 1.0, + "accuracy": 0.60673, + "precision_at_1": 0.63784, + "precision_at_3": 0.28028, + "precision_at_5": 0.18271, + "precision_at_10": 0.09912, + "precision_at_20": 0.05257, + "precision_at_100": 0.01117, + "precision_at_1000": 0.00113, + "mrr_at_1": 0.637845, + "mrr_at_3": 0.712615, + "mrr_at_5": 0.723517, + "mrr_at_10": 0.731, + "mrr_at_20": 0.733763, + "mrr_at_100": 0.734953, + "mrr_at_1000": 0.735008, + "nauc_ndcg_at_1_max": 0.448483, + "nauc_ndcg_at_1_std": -0.271013, + "nauc_ndcg_at_1_diff1": 0.703083, + "nauc_ndcg_at_3_max": 0.430961, + "nauc_ndcg_at_3_std": -0.30834, + "nauc_ndcg_at_3_diff1": 0.661292, + "nauc_ndcg_at_5_max": 0.461297, + "nauc_ndcg_at_5_std": -0.288896, + "nauc_ndcg_at_5_diff1": 0.657848, + "nauc_ndcg_at_10_max": 0.462291, + "nauc_ndcg_at_10_std": -0.289031, + "nauc_ndcg_at_10_diff1": 0.658077, + "nauc_ndcg_at_20_max": 0.46718, + "nauc_ndcg_at_20_std": -0.271868, + "nauc_ndcg_at_20_diff1": 0.664998, + "nauc_ndcg_at_100_max": 0.463538, + "nauc_ndcg_at_100_std": -0.273935, + "nauc_ndcg_at_100_diff1": 0.668808, + "nauc_ndcg_at_1000_max": 0.460959, + "nauc_ndcg_at_1000_std": -0.277313, + "nauc_ndcg_at_1000_diff1": 0.670018, + "nauc_map_at_1_max": 0.385951, + "nauc_map_at_1_std": -0.264102, + "nauc_map_at_1_diff1": 0.697433, + "nauc_map_at_3_max": 0.413124, + "nauc_map_at_3_std": -0.298718, + "nauc_map_at_3_diff1": 0.670678, + "nauc_map_at_5_max": 0.435063, + "nauc_map_at_5_std": -0.287976, + "nauc_map_at_5_diff1": 0.670466, + "nauc_map_at_10_max": 0.437614, + "nauc_map_at_10_std": -0.288733, + "nauc_map_at_10_diff1": 0.670485, + "nauc_map_at_20_max": 0.439981, + "nauc_map_at_20_std": -0.283556, + "nauc_map_at_20_diff1": 0.672888, + "nauc_map_at_100_max": 0.439532, + "nauc_map_at_100_std": -0.283761, + "nauc_map_at_100_diff1": 0.673339, + "nauc_map_at_1000_max": 0.43944, + "nauc_map_at_1000_std": -0.283867, + "nauc_map_at_1000_diff1": 0.673376, + "nauc_recall_at_1_max": 0.385951, + "nauc_recall_at_1_std": -0.264102, + "nauc_recall_at_1_diff1": 0.697433, + "nauc_recall_at_3_max": 0.416383, + "nauc_recall_at_3_std": -0.34683, + "nauc_recall_at_3_diff1": 0.61112, + "nauc_recall_at_5_max": 0.493603, + "nauc_recall_at_5_std": -0.296166, + "nauc_recall_at_5_diff1": 0.581893, + "nauc_recall_at_10_max": 0.507406, + "nauc_recall_at_10_std": -0.309266, + "nauc_recall_at_10_diff1": 0.555902, + "nauc_recall_at_20_max": 0.581124, + "nauc_recall_at_20_std": -0.141533, + "nauc_recall_at_20_diff1": 0.566391, + "nauc_recall_at_100_max": 0.759842, + "nauc_recall_at_100_std": 0.15091, + "nauc_recall_at_100_diff1": 0.51349, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.448483, + "nauc_precision_at_1_std": -0.271013, + "nauc_precision_at_1_diff1": 0.703083, + "nauc_precision_at_3_max": 0.471851, + "nauc_precision_at_3_std": -0.252573, + "nauc_precision_at_3_diff1": 0.432469, + "nauc_precision_at_5_max": 0.46336, + "nauc_precision_at_5_std": -0.122993, + "nauc_precision_at_5_diff1": 0.262711, + "nauc_precision_at_10_max": 0.377972, + "nauc_precision_at_10_std": -0.049899, + "nauc_precision_at_10_diff1": 0.083878, + "nauc_precision_at_20_max": 0.300412, + "nauc_precision_at_20_std": 0.090927, + "nauc_precision_at_20_diff1": -0.046504, + "nauc_precision_at_100_max": 0.167166, + "nauc_precision_at_100_std": 0.177123, + "nauc_precision_at_100_diff1": -0.214829, + "nauc_precision_at_1000_max": 0.123739, + "nauc_precision_at_1000_std": 0.181801, + "nauc_precision_at_1000_diff1": -0.250538, + "nauc_mrr_at_1_max": 0.448483, + "nauc_mrr_at_1_std": -0.271013, + "nauc_mrr_at_1_diff1": 0.703083, + "nauc_mrr_at_3_max": 0.472661, + "nauc_mrr_at_3_std": -0.294579, + "nauc_mrr_at_3_diff1": 0.677586, + "nauc_mrr_at_5_max": 0.480583, + "nauc_mrr_at_5_std": -0.281797, + "nauc_mrr_at_5_diff1": 0.673095, + "nauc_mrr_at_10_max": 0.478309, + "nauc_mrr_at_10_std": -0.280006, + "nauc_mrr_at_10_diff1": 0.674419, + "nauc_mrr_at_20_max": 0.477813, + "nauc_mrr_at_20_std": -0.277551, + "nauc_mrr_at_20_diff1": 0.676099, + "nauc_mrr_at_100_max": 0.477101, + "nauc_mrr_at_100_std": -0.278299, + "nauc_mrr_at_100_diff1": 0.676663, + "nauc_mrr_at_1000_max": 0.477012, + "nauc_mrr_at_1000_std": -0.27846, + "nauc_mrr_at_1000_diff1": 0.676722, + "hit_rate_at_1": 0.63784, + "hit_rate_at_3": 0.80827, + "hit_rate_at_5": 0.85464, + "hit_rate_at_10": 0.91103, + "hit_rate_at_20": 0.94987, + "hit_rate_at_100": 0.99123, + "hit_rate_at_1000": 1.0, + "main_score": 0.76211, + "hf_subset": "ar", + "languages": [ + "ara-Arab" + ] + }, + { + "ndcg_at_1": 0.77527, + "ndcg_at_3": 0.82215, + "ndcg_at_5": 0.8389, + "ndcg_at_10": 0.84975, + "ndcg_at_20": 0.85666, + "ndcg_at_100": 0.86487, + "ndcg_at_1000": 0.86645, + "map_at_1": 0.71783, + "map_at_3": 0.80096, + "map_at_5": 0.81193, + "map_at_10": 0.81778, + "map_at_20": 0.82003, + "map_at_100": 0.82142, + "map_at_1000": 0.82153, + "recall_at_1": 0.71783, + "recall_at_3": 0.8578, + "recall_at_5": 0.89579, + "recall_at_10": 0.925, + "recall_at_20": 0.94892, + "recall_at_100": 0.98996, + "recall_at_1000": 1.0, + "accuracy": 0.71783, + "precision_at_1": 0.77527, + "precision_at_3": 0.32437, + "precision_at_5": 0.20516, + "precision_at_10": 0.1071, + "precision_at_20": 0.05548, + "precision_at_100": 0.01167, + "precision_at_1000": 0.00118, + "mrr_at_1": 0.775269, + "mrr_at_3": 0.826882, + "mrr_at_5": 0.833925, + "mrr_at_10": 0.836988, + "mrr_at_20": 0.838737, + "mrr_at_100": 0.839491, + "mrr_at_1000": 0.839526, + "nauc_ndcg_at_1_max": 0.449169, + "nauc_ndcg_at_1_std": -0.512618, + "nauc_ndcg_at_1_diff1": 0.820063, + "nauc_ndcg_at_3_max": 0.444993, + "nauc_ndcg_at_3_std": -0.585662, + "nauc_ndcg_at_3_diff1": 0.786015, + "nauc_ndcg_at_5_max": 0.469326, + "nauc_ndcg_at_5_std": -0.586624, + "nauc_ndcg_at_5_diff1": 0.785359, + "nauc_ndcg_at_10_max": 0.467753, + "nauc_ndcg_at_10_std": -0.580762, + "nauc_ndcg_at_10_diff1": 0.787609, + "nauc_ndcg_at_20_max": 0.461384, + "nauc_ndcg_at_20_std": -0.57669, + "nauc_ndcg_at_20_diff1": 0.786055, + "nauc_ndcg_at_100_max": 0.451797, + "nauc_ndcg_at_100_std": -0.572867, + "nauc_ndcg_at_100_diff1": 0.789654, + "nauc_ndcg_at_1000_max": 0.454254, + "nauc_ndcg_at_1000_std": -0.566192, + "nauc_ndcg_at_1000_diff1": 0.790815, + "nauc_map_at_1_max": 0.325489, + "nauc_map_at_1_std": -0.472429, + "nauc_map_at_1_diff1": 0.792883, + "nauc_map_at_3_max": 0.422863, + "nauc_map_at_3_std": -0.558224, + "nauc_map_at_3_diff1": 0.788118, + "nauc_map_at_5_max": 0.441511, + "nauc_map_at_5_std": -0.557951, + "nauc_map_at_5_diff1": 0.788677, + "nauc_map_at_10_max": 0.44117, + "nauc_map_at_10_std": -0.555717, + "nauc_map_at_10_diff1": 0.789229, + "nauc_map_at_20_max": 0.438777, + "nauc_map_at_20_std": -0.555491, + "nauc_map_at_20_diff1": 0.788621, + "nauc_map_at_100_max": 0.437142, + "nauc_map_at_100_std": -0.554665, + "nauc_map_at_100_diff1": 0.789188, + "nauc_map_at_1000_max": 0.437313, + "nauc_map_at_1000_std": -0.554272, + "nauc_map_at_1000_diff1": 0.789251, + "nauc_recall_at_1_max": 0.325489, + "nauc_recall_at_1_std": -0.472429, + "nauc_recall_at_1_diff1": 0.792883, + "nauc_recall_at_3_max": 0.455071, + "nauc_recall_at_3_std": -0.660649, + "nauc_recall_at_3_diff1": 0.748342, + "nauc_recall_at_5_max": 0.529226, + "nauc_recall_at_5_std": -0.684013, + "nauc_recall_at_5_diff1": 0.731563, + "nauc_recall_at_10_max": 0.537011, + "nauc_recall_at_10_std": -0.682233, + "nauc_recall_at_10_diff1": 0.729358, + "nauc_recall_at_20_max": 0.527533, + "nauc_recall_at_20_std": -0.689444, + "nauc_recall_at_20_diff1": 0.708469, + "nauc_recall_at_100_max": 0.283697, + "nauc_recall_at_100_std": -1.076797, + "nauc_recall_at_100_diff1": 0.685749, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.449169, + "nauc_precision_at_1_std": -0.512618, + "nauc_precision_at_1_diff1": 0.820063, + "nauc_precision_at_3_max": 0.336026, + "nauc_precision_at_3_std": -0.254342, + "nauc_precision_at_3_diff1": 0.177372, + "nauc_precision_at_5_max": 0.304722, + "nauc_precision_at_5_std": -0.155577, + "nauc_precision_at_5_diff1": 0.038645, + "nauc_precision_at_10_max": 0.230245, + "nauc_precision_at_10_std": -0.04874, + "nauc_precision_at_10_diff1": -0.077717, + "nauc_precision_at_20_max": 0.152432, + "nauc_precision_at_20_std": 0.022496, + "nauc_precision_at_20_diff1": -0.190164, + "nauc_precision_at_100_max": 0.036813, + "nauc_precision_at_100_std": 0.15174, + "nauc_precision_at_100_diff1": -0.330675, + "nauc_precision_at_1000_max": 0.026921, + "nauc_precision_at_1000_std": 0.204662, + "nauc_precision_at_1000_diff1": -0.354116, + "nauc_mrr_at_1_max": 0.449169, + "nauc_mrr_at_1_std": -0.512618, + "nauc_mrr_at_1_diff1": 0.820063, + "nauc_mrr_at_3_max": 0.476076, + "nauc_mrr_at_3_std": -0.572697, + "nauc_mrr_at_3_diff1": 0.804514, + "nauc_mrr_at_5_max": 0.477562, + "nauc_mrr_at_5_std": -0.572847, + "nauc_mrr_at_5_diff1": 0.802546, + "nauc_mrr_at_10_max": 0.475925, + "nauc_mrr_at_10_std": -0.570208, + "nauc_mrr_at_10_diff1": 0.804213, + "nauc_mrr_at_20_max": 0.475036, + "nauc_mrr_at_20_std": -0.566944, + "nauc_mrr_at_20_diff1": 0.804186, + "nauc_mrr_at_100_max": 0.473968, + "nauc_mrr_at_100_std": -0.56615, + "nauc_mrr_at_100_diff1": 0.80457, + "nauc_mrr_at_1000_max": 0.473965, + "nauc_mrr_at_1000_std": -0.566007, + "nauc_mrr_at_1000_diff1": 0.80457, + "hit_rate_at_1": 0.77527, + "hit_rate_at_3": 0.8871, + "hit_rate_at_5": 0.91828, + "hit_rate_at_10": 0.93978, + "hit_rate_at_20": 0.96452, + "hit_rate_at_100": 0.99462, + "hit_rate_at_1000": 1.0, + "main_score": 0.84975, + "hf_subset": "zh", + "languages": [ + "zho-Hans" + ] + }, + { + "ndcg_at_1": 0.70915, + "ndcg_at_3": 0.77171, + "ndcg_at_5": 0.79247, + "ndcg_at_10": 0.80903, + "ndcg_at_20": 0.82035, + "ndcg_at_100": 0.82726, + "ndcg_at_1000": 0.82847, + "map_at_1": 0.68174, + "map_at_3": 0.75004, + "map_at_5": 0.76286, + "map_at_10": 0.77057, + "map_at_20": 0.77395, + "map_at_100": 0.77514, + "map_at_1000": 0.7752, + "recall_at_1": 0.68174, + "recall_at_3": 0.81852, + "recall_at_5": 0.86588, + "recall_at_10": 0.91367, + "recall_at_20": 0.95683, + "recall_at_100": 0.99126, + "recall_at_1000": 1.0, + "accuracy": 0.68174, + "precision_at_1": 0.70915, + "precision_at_3": 0.28948, + "precision_at_5": 0.18582, + "precision_at_10": 0.09877, + "precision_at_20": 0.052, + "precision_at_100": 0.01083, + "precision_at_1000": 0.00109, + "mrr_at_1": 0.709147, + "mrr_at_3": 0.766701, + "mrr_at_5": 0.776927, + "mrr_at_10": 0.783197, + "mrr_at_20": 0.785822, + "mrr_at_100": 0.786567, + "mrr_at_1000": 0.786609, + "nauc_ndcg_at_1_max": 0.441412, + "nauc_ndcg_at_1_std": -0.385014, + "nauc_ndcg_at_1_diff1": 0.791751, + "nauc_ndcg_at_3_max": 0.457277, + "nauc_ndcg_at_3_std": -0.444797, + "nauc_ndcg_at_3_diff1": 0.767093, + "nauc_ndcg_at_5_max": 0.464231, + "nauc_ndcg_at_5_std": -0.443696, + "nauc_ndcg_at_5_diff1": 0.769724, + "nauc_ndcg_at_10_max": 0.459379, + "nauc_ndcg_at_10_std": -0.453668, + "nauc_ndcg_at_10_diff1": 0.767852, + "nauc_ndcg_at_20_max": 0.460027, + "nauc_ndcg_at_20_std": -0.4314, + "nauc_ndcg_at_20_diff1": 0.770602, + "nauc_ndcg_at_100_max": 0.461346, + "nauc_ndcg_at_100_std": -0.423022, + "nauc_ndcg_at_100_diff1": 0.772218, + "nauc_ndcg_at_1000_max": 0.459707, + "nauc_ndcg_at_1000_std": -0.423733, + "nauc_ndcg_at_1000_diff1": 0.772549, + "nauc_map_at_1_max": 0.393082, + "nauc_map_at_1_std": -0.380323, + "nauc_map_at_1_diff1": 0.785259, + "nauc_map_at_3_max": 0.438841, + "nauc_map_at_3_std": -0.431543, + "nauc_map_at_3_diff1": 0.770637, + "nauc_map_at_5_max": 0.445685, + "nauc_map_at_5_std": -0.430897, + "nauc_map_at_5_diff1": 0.772338, + "nauc_map_at_10_max": 0.444578, + "nauc_map_at_10_std": -0.432763, + "nauc_map_at_10_diff1": 0.771302, + "nauc_map_at_20_max": 0.445503, + "nauc_map_at_20_std": -0.4269, + "nauc_map_at_20_diff1": 0.771947, + "nauc_map_at_100_max": 0.445776, + "nauc_map_at_100_std": -0.425674, + "nauc_map_at_100_diff1": 0.772335, + "nauc_map_at_1000_max": 0.445735, + "nauc_map_at_1000_std": -0.425627, + "nauc_map_at_1000_diff1": 0.772354, + "nauc_recall_at_1_max": 0.393082, + "nauc_recall_at_1_std": -0.380323, + "nauc_recall_at_1_diff1": 0.785259, + "nauc_recall_at_3_max": 0.47135, + "nauc_recall_at_3_std": -0.504878, + "nauc_recall_at_3_diff1": 0.734227, + "nauc_recall_at_5_max": 0.494529, + "nauc_recall_at_5_std": -0.527275, + "nauc_recall_at_5_diff1": 0.741435, + "nauc_recall_at_10_max": 0.466951, + "nauc_recall_at_10_std": -0.639145, + "nauc_recall_at_10_diff1": 0.714648, + "nauc_recall_at_20_max": 0.480286, + "nauc_recall_at_20_std": -0.52734, + "nauc_recall_at_20_diff1": 0.720754, + "nauc_recall_at_100_max": 0.693281, + "nauc_recall_at_100_std": -0.25957, + "nauc_recall_at_100_diff1": 0.736887, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.441412, + "nauc_precision_at_1_std": -0.385014, + "nauc_precision_at_1_diff1": 0.791751, + "nauc_precision_at_3_max": 0.440346, + "nauc_precision_at_3_std": -0.367996, + "nauc_precision_at_3_diff1": 0.489281, + "nauc_precision_at_5_max": 0.386652, + "nauc_precision_at_5_std": -0.265923, + "nauc_precision_at_5_diff1": 0.326505, + "nauc_precision_at_10_max": 0.289464, + "nauc_precision_at_10_std": -0.180023, + "nauc_precision_at_10_diff1": 0.129705, + "nauc_precision_at_20_max": 0.192126, + "nauc_precision_at_20_std": 0.046929, + "nauc_precision_at_20_diff1": -0.080581, + "nauc_precision_at_100_max": 0.086918, + "nauc_precision_at_100_std": 0.201542, + "nauc_precision_at_100_diff1": -0.256038, + "nauc_precision_at_1000_max": 0.049052, + "nauc_precision_at_1000_std": 0.229267, + "nauc_precision_at_1000_diff1": -0.304708, + "nauc_mrr_at_1_max": 0.441412, + "nauc_mrr_at_1_std": -0.385014, + "nauc_mrr_at_1_diff1": 0.791751, + "nauc_mrr_at_3_max": 0.477207, + "nauc_mrr_at_3_std": -0.415128, + "nauc_mrr_at_3_diff1": 0.77668, + "nauc_mrr_at_5_max": 0.475962, + "nauc_mrr_at_5_std": -0.412221, + "nauc_mrr_at_5_diff1": 0.777623, + "nauc_mrr_at_10_max": 0.473442, + "nauc_mrr_at_10_std": -0.415537, + "nauc_mrr_at_10_diff1": 0.777367, + "nauc_mrr_at_20_max": 0.472655, + "nauc_mrr_at_20_std": -0.410678, + "nauc_mrr_at_20_diff1": 0.778388, + "nauc_mrr_at_100_max": 0.472705, + "nauc_mrr_at_100_std": -0.409851, + "nauc_mrr_at_100_diff1": 0.778324, + "nauc_mrr_at_1000_max": 0.472635, + "nauc_mrr_at_1000_std": -0.409888, + "nauc_mrr_at_1000_diff1": 0.778337, + "hit_rate_at_1": 0.70915, + "hit_rate_at_3": 0.83762, + "hit_rate_at_5": 0.88181, + "hit_rate_at_10": 0.92703, + "hit_rate_at_20": 0.96608, + "hit_rate_at_100": 0.99281, + "hit_rate_at_1000": 1.0, + "main_score": 0.80903, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.66905, + "ndcg_at_3": 0.76141, + "ndcg_at_5": 0.78285, + "ndcg_at_10": 0.79905, + "ndcg_at_20": 0.80784, + "ndcg_at_100": 0.81533, + "ndcg_at_1000": 0.81608, + "map_at_1": 0.64896, + "map_at_3": 0.73493, + "map_at_5": 0.7478, + "map_at_10": 0.75542, + "map_at_20": 0.75797, + "map_at_100": 0.7592, + "map_at_1000": 0.75923, + "recall_at_1": 0.64896, + "recall_at_3": 0.82652, + "recall_at_5": 0.87657, + "recall_at_10": 0.92305, + "recall_at_20": 0.95659, + "recall_at_100": 0.99438, + "recall_at_1000": 1.0, + "accuracy": 0.64896, + "precision_at_1": 0.66905, + "precision_at_3": 0.28975, + "precision_at_5": 0.18529, + "precision_at_10": 0.09837, + "precision_at_20": 0.05123, + "precision_at_100": 0.01066, + "precision_at_1000": 0.00107, + "mrr_at_1": 0.66905, + "mrr_at_3": 0.74617, + "mrr_at_5": 0.756895, + "mrr_at_10": 0.763061, + "mrr_at_20": 0.765193, + "mrr_at_100": 0.766225, + "mrr_at_1000": 0.766254, + "nauc_ndcg_at_1_max": 0.424199, + "nauc_ndcg_at_1_std": -0.205804, + "nauc_ndcg_at_1_diff1": 0.749937, + "nauc_ndcg_at_3_max": 0.480662, + "nauc_ndcg_at_3_std": -0.269244, + "nauc_ndcg_at_3_diff1": 0.729902, + "nauc_ndcg_at_5_max": 0.481297, + "nauc_ndcg_at_5_std": -0.266791, + "nauc_ndcg_at_5_diff1": 0.730342, + "nauc_ndcg_at_10_max": 0.478689, + "nauc_ndcg_at_10_std": -0.263023, + "nauc_ndcg_at_10_diff1": 0.731569, + "nauc_ndcg_at_20_max": 0.480449, + "nauc_ndcg_at_20_std": -0.254061, + "nauc_ndcg_at_20_diff1": 0.732995, + "nauc_ndcg_at_100_max": 0.471819, + "nauc_ndcg_at_100_std": -0.242621, + "nauc_ndcg_at_100_diff1": 0.736453, + "nauc_ndcg_at_1000_max": 0.469763, + "nauc_ndcg_at_1000_std": -0.245489, + "nauc_ndcg_at_1000_diff1": 0.735953, + "nauc_map_at_1_max": 0.391961, + "nauc_map_at_1_std": -0.220496, + "nauc_map_at_1_diff1": 0.738894, + "nauc_map_at_3_max": 0.46118, + "nauc_map_at_3_std": -0.255292, + "nauc_map_at_3_diff1": 0.730852, + "nauc_map_at_5_max": 0.461346, + "nauc_map_at_5_std": -0.255122, + "nauc_map_at_5_diff1": 0.73135, + "nauc_map_at_10_max": 0.460459, + "nauc_map_at_10_std": -0.252182, + "nauc_map_at_10_diff1": 0.732353, + "nauc_map_at_20_max": 0.461339, + "nauc_map_at_20_std": -0.248872, + "nauc_map_at_20_diff1": 0.732522, + "nauc_map_at_100_max": 0.460045, + "nauc_map_at_100_std": -0.247709, + "nauc_map_at_100_diff1": 0.733049, + "nauc_map_at_1000_max": 0.459979, + "nauc_map_at_1000_std": -0.247783, + "nauc_map_at_1000_diff1": 0.733028, + "nauc_recall_at_1_max": 0.391961, + "nauc_recall_at_1_std": -0.220496, + "nauc_recall_at_1_diff1": 0.738894, + "nauc_recall_at_3_max": 0.539464, + "nauc_recall_at_3_std": -0.339383, + "nauc_recall_at_3_diff1": 0.700233, + "nauc_recall_at_5_max": 0.561926, + "nauc_recall_at_5_std": -0.355205, + "nauc_recall_at_5_diff1": 0.694521, + "nauc_recall_at_10_max": 0.590599, + "nauc_recall_at_10_std": -0.399917, + "nauc_recall_at_10_diff1": 0.688958, + "nauc_recall_at_20_max": 0.687506, + "nauc_recall_at_20_std": -0.397951, + "nauc_recall_at_20_diff1": 0.677769, + "nauc_recall_at_100_max": 0.908943, + "nauc_recall_at_100_std": 0.404331, + "nauc_recall_at_100_diff1": 0.827857, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.424199, + "nauc_precision_at_1_std": -0.205804, + "nauc_precision_at_1_diff1": 0.749937, + "nauc_precision_at_3_max": 0.486214, + "nauc_precision_at_3_std": -0.210951, + "nauc_precision_at_3_diff1": 0.497881, + "nauc_precision_at_5_max": 0.428694, + "nauc_precision_at_5_std": -0.147707, + "nauc_precision_at_5_diff1": 0.359592, + "nauc_precision_at_10_max": 0.320067, + "nauc_precision_at_10_std": -0.050148, + "nauc_precision_at_10_diff1": 0.175346, + "nauc_precision_at_20_max": 0.236393, + "nauc_precision_at_20_std": 0.076491, + "nauc_precision_at_20_diff1": 0.013378, + "nauc_precision_at_100_max": 0.072715, + "nauc_precision_at_100_std": 0.260886, + "nauc_precision_at_100_diff1": -0.180219, + "nauc_precision_at_1000_max": 0.031893, + "nauc_precision_at_1000_std": 0.249268, + "nauc_precision_at_1000_diff1": -0.224112, + "nauc_mrr_at_1_max": 0.424199, + "nauc_mrr_at_1_std": -0.205804, + "nauc_mrr_at_1_diff1": 0.749937, + "nauc_mrr_at_3_max": 0.468467, + "nauc_mrr_at_3_std": -0.240538, + "nauc_mrr_at_3_diff1": 0.742952, + "nauc_mrr_at_5_max": 0.467213, + "nauc_mrr_at_5_std": -0.236234, + "nauc_mrr_at_5_diff1": 0.743632, + "nauc_mrr_at_10_max": 0.465125, + "nauc_mrr_at_10_std": -0.234436, + "nauc_mrr_at_10_diff1": 0.743531, + "nauc_mrr_at_20_max": 0.465364, + "nauc_mrr_at_20_std": -0.233641, + "nauc_mrr_at_20_diff1": 0.744419, + "nauc_mrr_at_100_max": 0.463932, + "nauc_mrr_at_100_std": -0.232875, + "nauc_mrr_at_100_diff1": 0.744521, + "nauc_mrr_at_1000_max": 0.463867, + "nauc_mrr_at_1000_std": -0.232942, + "nauc_mrr_at_1000_diff1": 0.7445, + "hit_rate_at_1": 0.66905, + "hit_rate_at_3": 0.84065, + "hit_rate_at_5": 0.88662, + "hit_rate_at_10": 0.92952, + "hit_rate_at_20": 0.96118, + "hit_rate_at_100": 0.99489, + "hit_rate_at_1000": 1.0, + "main_score": 0.79905, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.67161, + "ndcg_at_3": 0.75422, + "ndcg_at_5": 0.77002, + "ndcg_at_10": 0.7836, + "ndcg_at_20": 0.79334, + "ndcg_at_100": 0.80299, + "ndcg_at_1000": 0.80609, + "map_at_1": 0.65042, + "map_at_3": 0.73117, + "map_at_5": 0.74035, + "map_at_10": 0.74651, + "map_at_20": 0.74942, + "map_at_100": 0.75078, + "map_at_1000": 0.75094, + "recall_at_1": 0.65042, + "recall_at_3": 0.81321, + "recall_at_5": 0.85011, + "recall_at_10": 0.89018, + "recall_at_20": 0.92691, + "recall_at_100": 0.97802, + "recall_at_1000": 1.0, + "accuracy": 0.65042, + "precision_at_1": 0.67161, + "precision_at_3": 0.28496, + "precision_at_5": 0.17966, + "precision_at_10": 0.09449, + "precision_at_20": 0.04936, + "precision_at_100": 0.01047, + "precision_at_1000": 0.00107, + "mrr_at_1": 0.67161, + "mrr_at_3": 0.739054, + "mrr_at_5": 0.747369, + "mrr_at_10": 0.752463, + "mrr_at_20": 0.755015, + "mrr_at_100": 0.756202, + "mrr_at_1000": 0.756322, + "nauc_ndcg_at_1_max": 0.448914, + "nauc_ndcg_at_1_std": -0.324926, + "nauc_ndcg_at_1_diff1": 0.774517, + "nauc_ndcg_at_3_max": 0.452749, + "nauc_ndcg_at_3_std": -0.442445, + "nauc_ndcg_at_3_diff1": 0.735184, + "nauc_ndcg_at_5_max": 0.459621, + "nauc_ndcg_at_5_std": -0.4546, + "nauc_ndcg_at_5_diff1": 0.735886, + "nauc_ndcg_at_10_max": 0.457555, + "nauc_ndcg_at_10_std": -0.461222, + "nauc_ndcg_at_10_diff1": 0.738014, + "nauc_ndcg_at_20_max": 0.455353, + "nauc_ndcg_at_20_std": -0.462713, + "nauc_ndcg_at_20_diff1": 0.738239, + "nauc_ndcg_at_100_max": 0.457143, + "nauc_ndcg_at_100_std": -0.435231, + "nauc_ndcg_at_100_diff1": 0.744643, + "nauc_ndcg_at_1000_max": 0.455811, + "nauc_ndcg_at_1000_std": -0.430815, + "nauc_ndcg_at_1000_diff1": 0.74449, + "nauc_map_at_1_max": 0.398571, + "nauc_map_at_1_std": -0.318271, + "nauc_map_at_1_diff1": 0.761939, + "nauc_map_at_3_max": 0.443397, + "nauc_map_at_3_std": -0.407243, + "nauc_map_at_3_diff1": 0.740855, + "nauc_map_at_5_max": 0.447427, + "nauc_map_at_5_std": -0.412628, + "nauc_map_at_5_diff1": 0.740864, + "nauc_map_at_10_max": 0.447185, + "nauc_map_at_10_std": -0.413569, + "nauc_map_at_10_diff1": 0.741745, + "nauc_map_at_20_max": 0.446647, + "nauc_map_at_20_std": -0.413744, + "nauc_map_at_20_diff1": 0.742054, + "nauc_map_at_100_max": 0.447003, + "nauc_map_at_100_std": -0.410959, + "nauc_map_at_100_diff1": 0.743061, + "nauc_map_at_1000_max": 0.446983, + "nauc_map_at_1000_std": -0.410747, + "nauc_map_at_1000_diff1": 0.743073, + "nauc_recall_at_1_max": 0.398571, + "nauc_recall_at_1_std": -0.318271, + "nauc_recall_at_1_diff1": 0.761939, + "nauc_recall_at_3_max": 0.459157, + "nauc_recall_at_3_std": -0.569934, + "nauc_recall_at_3_diff1": 0.692566, + "nauc_recall_at_5_max": 0.485709, + "nauc_recall_at_5_std": -0.637688, + "nauc_recall_at_5_diff1": 0.690177, + "nauc_recall_at_10_max": 0.480863, + "nauc_recall_at_10_std": -0.736245, + "nauc_recall_at_10_diff1": 0.691082, + "nauc_recall_at_20_max": 0.469141, + "nauc_recall_at_20_std": -0.878043, + "nauc_recall_at_20_diff1": 0.667247, + "nauc_recall_at_100_max": 0.544912, + "nauc_recall_at_100_std": -0.659421, + "nauc_recall_at_100_diff1": 0.751116, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.448914, + "nauc_precision_at_1_std": -0.324926, + "nauc_precision_at_1_diff1": 0.774517, + "nauc_precision_at_3_max": 0.431176, + "nauc_precision_at_3_std": -0.435372, + "nauc_precision_at_3_diff1": 0.48291, + "nauc_precision_at_5_max": 0.409565, + "nauc_precision_at_5_std": -0.4316, + "nauc_precision_at_5_diff1": 0.390297, + "nauc_precision_at_10_max": 0.34179, + "nauc_precision_at_10_std": -0.400005, + "nauc_precision_at_10_diff1": 0.277453, + "nauc_precision_at_20_max": 0.265544, + "nauc_precision_at_20_std": -0.346952, + "nauc_precision_at_20_diff1": 0.133976, + "nauc_precision_at_100_max": 0.15244, + "nauc_precision_at_100_std": -0.025393, + "nauc_precision_at_100_diff1": -0.097123, + "nauc_precision_at_1000_max": 0.069057, + "nauc_precision_at_1000_std": 0.131495, + "nauc_precision_at_1000_diff1": -0.245314, + "nauc_mrr_at_1_max": 0.448914, + "nauc_mrr_at_1_std": -0.324926, + "nauc_mrr_at_1_diff1": 0.774517, + "nauc_mrr_at_3_max": 0.463906, + "nauc_mrr_at_3_std": -0.40829, + "nauc_mrr_at_3_diff1": 0.752947, + "nauc_mrr_at_5_max": 0.465571, + "nauc_mrr_at_5_std": -0.413382, + "nauc_mrr_at_5_diff1": 0.754601, + "nauc_mrr_at_10_max": 0.464371, + "nauc_mrr_at_10_std": -0.415054, + "nauc_mrr_at_10_diff1": 0.755645, + "nauc_mrr_at_20_max": 0.463847, + "nauc_mrr_at_20_std": -0.414569, + "nauc_mrr_at_20_diff1": 0.755791, + "nauc_mrr_at_100_max": 0.464197, + "nauc_mrr_at_100_std": -0.411334, + "nauc_mrr_at_100_diff1": 0.7564, + "nauc_mrr_at_1000_max": 0.464079, + "nauc_mrr_at_1000_std": -0.41128, + "nauc_mrr_at_1000_diff1": 0.756345, + "hit_rate_at_1": 0.67161, + "hit_rate_at_3": 0.82309, + "hit_rate_at_5": 0.85911, + "hit_rate_at_10": 0.89619, + "hit_rate_at_20": 0.93114, + "hit_rate_at_100": 0.98093, + "hit_rate_at_1000": 1.0, + "main_score": 0.7836, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ] + }, + { + "ndcg_at_1": 0.55659, + "ndcg_at_3": 0.63155, + "ndcg_at_5": 0.65354, + "ndcg_at_10": 0.6675, + "ndcg_at_20": 0.68253, + "ndcg_at_100": 0.69698, + "ndcg_at_1000": 0.70833, + "map_at_1": 0.53634, + "map_at_3": 0.60765, + "map_at_5": 0.62008, + "map_at_10": 0.62592, + "map_at_20": 0.63036, + "map_at_100": 0.63231, + "map_at_1000": 0.63283, + "recall_at_1": 0.53634, + "recall_at_3": 0.68795, + "recall_at_5": 0.74005, + "recall_at_10": 0.78262, + "recall_at_20": 0.83939, + "recall_at_100": 0.9171, + "recall_at_1000": 1.0, + "accuracy": 0.53634, + "precision_at_1": 0.55659, + "precision_at_3": 0.23988, + "precision_at_5": 0.15556, + "precision_at_10": 0.08235, + "precision_at_20": 0.04444, + "precision_at_100": 0.00978, + "precision_at_1000": 0.00108, + "mrr_at_1": 0.556594, + "mrr_at_3": 0.621668, + "mrr_at_5": 0.633766, + "mrr_at_10": 0.639453, + "mrr_at_20": 0.643468, + "mrr_at_100": 0.645128, + "mrr_at_1000": 0.645541, + "nauc_ndcg_at_1_max": 0.470229, + "nauc_ndcg_at_1_std": -0.376471, + "nauc_ndcg_at_1_diff1": 0.77644, + "nauc_ndcg_at_3_max": 0.463089, + "nauc_ndcg_at_3_std": -0.407303, + "nauc_ndcg_at_3_diff1": 0.720675, + "nauc_ndcg_at_5_max": 0.471745, + "nauc_ndcg_at_5_std": -0.407459, + "nauc_ndcg_at_5_diff1": 0.714957, + "nauc_ndcg_at_10_max": 0.481449, + "nauc_ndcg_at_10_std": -0.409785, + "nauc_ndcg_at_10_diff1": 0.715993, + "nauc_ndcg_at_20_max": 0.483692, + "nauc_ndcg_at_20_std": -0.402849, + "nauc_ndcg_at_20_diff1": 0.718874, + "nauc_ndcg_at_100_max": 0.480741, + "nauc_ndcg_at_100_std": -0.398882, + "nauc_ndcg_at_100_diff1": 0.721397, + "nauc_ndcg_at_1000_max": 0.478217, + "nauc_ndcg_at_1000_std": -0.399433, + "nauc_ndcg_at_1000_diff1": 0.726374, + "nauc_map_at_1_max": 0.451307, + "nauc_map_at_1_std": -0.361201, + "nauc_map_at_1_diff1": 0.769202, + "nauc_map_at_3_max": 0.461227, + "nauc_map_at_3_std": -0.398722, + "nauc_map_at_3_diff1": 0.732364, + "nauc_map_at_5_max": 0.467305, + "nauc_map_at_5_std": -0.398218, + "nauc_map_at_5_diff1": 0.73018, + "nauc_map_at_10_max": 0.471477, + "nauc_map_at_10_std": -0.398545, + "nauc_map_at_10_diff1": 0.730388, + "nauc_map_at_20_max": 0.471855, + "nauc_map_at_20_std": -0.396952, + "nauc_map_at_20_diff1": 0.731161, + "nauc_map_at_100_max": 0.471717, + "nauc_map_at_100_std": -0.39657, + "nauc_map_at_100_diff1": 0.731672, + "nauc_map_at_1000_max": 0.471648, + "nauc_map_at_1000_std": -0.396543, + "nauc_map_at_1000_diff1": 0.731886, + "nauc_recall_at_1_max": 0.451307, + "nauc_recall_at_1_std": -0.361201, + "nauc_recall_at_1_diff1": 0.769202, + "nauc_recall_at_3_max": 0.454695, + "nauc_recall_at_3_std": -0.431205, + "nauc_recall_at_3_diff1": 0.66988, + "nauc_recall_at_5_max": 0.472278, + "nauc_recall_at_5_std": -0.43656, + "nauc_recall_at_5_diff1": 0.640151, + "nauc_recall_at_10_max": 0.511611, + "nauc_recall_at_10_std": -0.451253, + "nauc_recall_at_10_diff1": 0.637287, + "nauc_recall_at_20_max": 0.532621, + "nauc_recall_at_20_std": -0.414227, + "nauc_recall_at_20_diff1": 0.628121, + "nauc_recall_at_100_max": 0.52263, + "nauc_recall_at_100_std": -0.369694, + "nauc_recall_at_100_diff1": 0.597555, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.470229, + "nauc_precision_at_1_std": -0.376471, + "nauc_precision_at_1_diff1": 0.77644, + "nauc_precision_at_3_max": 0.438192, + "nauc_precision_at_3_std": -0.418554, + "nauc_precision_at_3_diff1": 0.598866, + "nauc_precision_at_5_max": 0.440654, + "nauc_precision_at_5_std": -0.397466, + "nauc_precision_at_5_diff1": 0.536662, + "nauc_precision_at_10_max": 0.464538, + "nauc_precision_at_10_std": -0.392218, + "nauc_precision_at_10_diff1": 0.502025, + "nauc_precision_at_20_max": 0.417953, + "nauc_precision_at_20_std": -0.295229, + "nauc_precision_at_20_diff1": 0.394683, + "nauc_precision_at_100_max": 0.267395, + "nauc_precision_at_100_std": -0.137928, + "nauc_precision_at_100_diff1": 0.182176, + "nauc_precision_at_1000_max": -0.056999, + "nauc_precision_at_1000_std": 0.097913, + "nauc_precision_at_1000_diff1": -0.214817, + "nauc_mrr_at_1_max": 0.470229, + "nauc_mrr_at_1_std": -0.376471, + "nauc_mrr_at_1_diff1": 0.77644, + "nauc_mrr_at_3_max": 0.471763, + "nauc_mrr_at_3_std": -0.400184, + "nauc_mrr_at_3_diff1": 0.739612, + "nauc_mrr_at_5_max": 0.474604, + "nauc_mrr_at_5_std": -0.39895, + "nauc_mrr_at_5_diff1": 0.736668, + "nauc_mrr_at_10_max": 0.477383, + "nauc_mrr_at_10_std": -0.400586, + "nauc_mrr_at_10_diff1": 0.736929, + "nauc_mrr_at_20_max": 0.477868, + "nauc_mrr_at_20_std": -0.399059, + "nauc_mrr_at_20_diff1": 0.738188, + "nauc_mrr_at_100_max": 0.477604, + "nauc_mrr_at_100_std": -0.398424, + "nauc_mrr_at_100_diff1": 0.738232, + "nauc_mrr_at_1000_max": 0.47762, + "nauc_mrr_at_1000_std": -0.398462, + "nauc_mrr_at_1000_diff1": 0.738491, + "hit_rate_at_1": 0.55659, + "hit_rate_at_3": 0.70301, + "hit_rate_at_5": 0.75597, + "hit_rate_at_10": 0.79855, + "hit_rate_at_20": 0.85358, + "hit_rate_at_100": 0.92731, + "hit_rate_at_1000": 1.0, + "main_score": 0.6675, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.74447, + "ndcg_at_3": 0.81852, + "ndcg_at_5": 0.83828, + "ndcg_at_10": 0.85076, + "ndcg_at_20": 0.85585, + "ndcg_at_100": 0.86055, + "ndcg_at_1000": 0.86069, + "map_at_1": 0.72577, + "map_at_3": 0.79716, + "map_at_5": 0.80879, + "map_at_10": 0.81426, + "map_at_20": 0.81601, + "map_at_100": 0.81677, + "map_at_1000": 0.81678, + "recall_at_1": 0.72577, + "recall_at_3": 0.87173, + "recall_at_5": 0.91885, + "recall_at_10": 0.95674, + "recall_at_20": 0.9751, + "recall_at_100": 0.99899, + "recall_at_1000": 1.0, + "accuracy": 0.72577, + "precision_at_1": 0.74447, + "precision_at_3": 0.30449, + "precision_at_5": 0.19316, + "precision_at_10": 0.10091, + "precision_at_20": 0.05161, + "precision_at_100": 0.01059, + "precision_at_1000": 0.00106, + "mrr_at_1": 0.744467, + "mrr_at_3": 0.805332, + "mrr_at_5": 0.815493, + "mrr_at_10": 0.820036, + "mrr_at_20": 0.821327, + "mrr_at_100": 0.821942, + "mrr_at_1000": 0.821947, + "nauc_ndcg_at_1_max": 0.48382, + "nauc_ndcg_at_1_std": -0.349205, + "nauc_ndcg_at_1_diff1": 0.81683, + "nauc_ndcg_at_3_max": 0.512339, + "nauc_ndcg_at_3_std": -0.393315, + "nauc_ndcg_at_3_diff1": 0.78214, + "nauc_ndcg_at_5_max": 0.520409, + "nauc_ndcg_at_5_std": -0.383895, + "nauc_ndcg_at_5_diff1": 0.783567, + "nauc_ndcg_at_10_max": 0.517601, + "nauc_ndcg_at_10_std": -0.374374, + "nauc_ndcg_at_10_diff1": 0.790237, + "nauc_ndcg_at_20_max": 0.513734, + "nauc_ndcg_at_20_std": -0.372222, + "nauc_ndcg_at_20_diff1": 0.789054, + "nauc_ndcg_at_100_max": 0.510037, + "nauc_ndcg_at_100_std": -0.372367, + "nauc_ndcg_at_100_diff1": 0.79295, + "nauc_ndcg_at_1000_max": 0.509991, + "nauc_ndcg_at_1000_std": -0.37155, + "nauc_ndcg_at_1000_diff1": 0.79287, + "nauc_map_at_1_max": 0.450741, + "nauc_map_at_1_std": -0.355686, + "nauc_map_at_1_diff1": 0.807397, + "nauc_map_at_3_max": 0.500871, + "nauc_map_at_3_std": -0.388655, + "nauc_map_at_3_diff1": 0.791355, + "nauc_map_at_5_max": 0.505568, + "nauc_map_at_5_std": -0.382875, + "nauc_map_at_5_diff1": 0.792667, + "nauc_map_at_10_max": 0.503938, + "nauc_map_at_10_std": -0.379473, + "nauc_map_at_10_diff1": 0.795052, + "nauc_map_at_20_max": 0.50341, + "nauc_map_at_20_std": -0.377263, + "nauc_map_at_20_diff1": 0.794511, + "nauc_map_at_100_max": 0.503319, + "nauc_map_at_100_std": -0.376822, + "nauc_map_at_100_diff1": 0.795098, + "nauc_map_at_1000_max": 0.503318, + "nauc_map_at_1000_std": -0.376797, + "nauc_map_at_1000_diff1": 0.795096, + "nauc_recall_at_1_max": 0.450741, + "nauc_recall_at_1_std": -0.355686, + "nauc_recall_at_1_diff1": 0.807397, + "nauc_recall_at_3_max": 0.543294, + "nauc_recall_at_3_std": -0.437326, + "nauc_recall_at_3_diff1": 0.737482, + "nauc_recall_at_5_max": 0.589524, + "nauc_recall_at_5_std": -0.416321, + "nauc_recall_at_5_diff1": 0.716274, + "nauc_recall_at_10_max": 0.628347, + "nauc_recall_at_10_std": -0.368573, + "nauc_recall_at_10_diff1": 0.733918, + "nauc_recall_at_20_max": 0.628204, + "nauc_recall_at_20_std": -0.356622, + "nauc_recall_at_20_diff1": 0.683991, + "nauc_recall_at_100_max": 0.554225, + "nauc_recall_at_100_std": -1.151166, + "nauc_recall_at_100_diff1": 0.869339, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.48382, + "nauc_precision_at_1_std": -0.349205, + "nauc_precision_at_1_diff1": 0.81683, + "nauc_precision_at_3_max": 0.412819, + "nauc_precision_at_3_std": -0.234363, + "nauc_precision_at_3_diff1": 0.39725, + "nauc_precision_at_5_max": 0.356925, + "nauc_precision_at_5_std": -0.10382, + "nauc_precision_at_5_diff1": 0.218973, + "nauc_precision_at_10_max": 0.242781, + "nauc_precision_at_10_std": 0.059323, + "nauc_precision_at_10_diff1": 0.015632, + "nauc_precision_at_20_max": 0.145227, + "nauc_precision_at_20_std": 0.167904, + "nauc_precision_at_20_diff1": -0.144018, + "nauc_precision_at_100_max": 0.010195, + "nauc_precision_at_100_std": 0.282189, + "nauc_precision_at_100_diff1": -0.327435, + "nauc_precision_at_1000_max": 0.004989, + "nauc_precision_at_1000_std": 0.295905, + "nauc_precision_at_1000_diff1": -0.338887, + "nauc_mrr_at_1_max": 0.48382, + "nauc_mrr_at_1_std": -0.349205, + "nauc_mrr_at_1_diff1": 0.81683, + "nauc_mrr_at_3_max": 0.509216, + "nauc_mrr_at_3_std": -0.36985, + "nauc_mrr_at_3_diff1": 0.793855, + "nauc_mrr_at_5_max": 0.510545, + "nauc_mrr_at_5_std": -0.366212, + "nauc_mrr_at_5_diff1": 0.794597, + "nauc_mrr_at_10_max": 0.508989, + "nauc_mrr_at_10_std": -0.362686, + "nauc_mrr_at_10_diff1": 0.796793, + "nauc_mrr_at_20_max": 0.507718, + "nauc_mrr_at_20_std": -0.363672, + "nauc_mrr_at_20_diff1": 0.796864, + "nauc_mrr_at_100_max": 0.50723, + "nauc_mrr_at_100_std": -0.363783, + "nauc_mrr_at_100_diff1": 0.797361, + "nauc_mrr_at_1000_max": 0.507229, + "nauc_mrr_at_1000_std": -0.363757, + "nauc_mrr_at_1000_diff1": 0.797359, + "hit_rate_at_1": 0.74447, + "hit_rate_at_3": 0.88129, + "hit_rate_at_5": 0.92555, + "hit_rate_at_10": 0.96076, + "hit_rate_at_20": 0.97787, + "hit_rate_at_100": 0.99899, + "hit_rate_at_1000": 1.0, + "main_score": 0.85076, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ] + }, + { + "ndcg_at_1": 0.39958, + "ndcg_at_3": 0.47543, + "ndcg_at_5": 0.50423, + "ndcg_at_10": 0.53136, + "ndcg_at_20": 0.55007, + "ndcg_at_100": 0.57612, + "ndcg_at_1000": 0.59108, + "map_at_1": 0.38137, + "map_at_3": 0.45031, + "map_at_5": 0.46742, + "map_at_10": 0.47918, + "map_at_20": 0.48456, + "map_at_100": 0.48839, + "map_at_1000": 0.48901, + "recall_at_1": 0.38137, + "recall_at_3": 0.53087, + "recall_at_5": 0.59834, + "recall_at_10": 0.67967, + "recall_at_20": 0.7513, + "recall_at_100": 0.8871, + "recall_at_1000": 1.0, + "accuracy": 0.38137, + "precision_at_1": 0.39958, + "precision_at_3": 0.19008, + "precision_at_5": 0.12986, + "precision_at_10": 0.07388, + "precision_at_20": 0.0411, + "precision_at_100": 0.00981, + "precision_at_1000": 0.00111, + "mrr_at_1": 0.399584, + "mrr_at_3": 0.468089, + "mrr_at_5": 0.482501, + "mrr_at_10": 0.493376, + "mrr_at_20": 0.498152, + "mrr_at_100": 0.501375, + "mrr_at_1000": 0.501918, + "nauc_ndcg_at_1_max": 0.370234, + "nauc_ndcg_at_1_std": 0.01079, + "nauc_ndcg_at_1_diff1": 0.588242, + "nauc_ndcg_at_3_max": 0.375483, + "nauc_ndcg_at_3_std": -0.003995, + "nauc_ndcg_at_3_diff1": 0.54526, + "nauc_ndcg_at_5_max": 0.377091, + "nauc_ndcg_at_5_std": -0.003772, + "nauc_ndcg_at_5_diff1": 0.533043, + "nauc_ndcg_at_10_max": 0.388284, + "nauc_ndcg_at_10_std": 0.003626, + "nauc_ndcg_at_10_diff1": 0.530986, + "nauc_ndcg_at_20_max": 0.384189, + "nauc_ndcg_at_20_std": 0.007022, + "nauc_ndcg_at_20_diff1": 0.526917, + "nauc_ndcg_at_100_max": 0.385284, + "nauc_ndcg_at_100_std": 0.010598, + "nauc_ndcg_at_100_diff1": 0.534694, + "nauc_ndcg_at_1000_max": 0.3821, + "nauc_ndcg_at_1000_std": 0.004774, + "nauc_ndcg_at_1000_diff1": 0.539937, + "nauc_map_at_1_max": 0.356256, + "nauc_map_at_1_std": 0.011902, + "nauc_map_at_1_diff1": 0.593818, + "nauc_map_at_3_max": 0.37361, + "nauc_map_at_3_std": -0.001272, + "nauc_map_at_3_diff1": 0.55609, + "nauc_map_at_5_max": 0.375435, + "nauc_map_at_5_std": -0.000424, + "nauc_map_at_5_diff1": 0.549134, + "nauc_map_at_10_max": 0.379374, + "nauc_map_at_10_std": 0.002933, + "nauc_map_at_10_diff1": 0.547751, + "nauc_map_at_20_max": 0.378232, + "nauc_map_at_20_std": 0.003646, + "nauc_map_at_20_diff1": 0.546864, + "nauc_map_at_100_max": 0.378291, + "nauc_map_at_100_std": 0.003964, + "nauc_map_at_100_diff1": 0.548067, + "nauc_map_at_1000_max": 0.378182, + "nauc_map_at_1000_std": 0.003745, + "nauc_map_at_1000_diff1": 0.548315, + "nauc_recall_at_1_max": 0.356256, + "nauc_recall_at_1_std": 0.011902, + "nauc_recall_at_1_diff1": 0.593818, + "nauc_recall_at_3_max": 0.368225, + "nauc_recall_at_3_std": -0.012399, + "nauc_recall_at_3_diff1": 0.504083, + "nauc_recall_at_5_max": 0.373067, + "nauc_recall_at_5_std": -0.011384, + "nauc_recall_at_5_diff1": 0.47082, + "nauc_recall_at_10_max": 0.41497, + "nauc_recall_at_10_std": 0.014506, + "nauc_recall_at_10_diff1": 0.456829, + "nauc_recall_at_20_max": 0.398879, + "nauc_recall_at_20_std": 0.036608, + "nauc_recall_at_20_diff1": 0.419626, + "nauc_recall_at_100_max": 0.438455, + "nauc_recall_at_100_std": 0.118527, + "nauc_recall_at_100_diff1": 0.431786, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.370234, + "nauc_precision_at_1_std": 0.01079, + "nauc_precision_at_1_diff1": 0.588242, + "nauc_precision_at_3_max": 0.394783, + "nauc_precision_at_3_std": -0.006268, + "nauc_precision_at_3_diff1": 0.470689, + "nauc_precision_at_5_max": 0.372351, + "nauc_precision_at_5_std": -0.003664, + "nauc_precision_at_5_diff1": 0.398326, + "nauc_precision_at_10_max": 0.381064, + "nauc_precision_at_10_std": 0.02709, + "nauc_precision_at_10_diff1": 0.34298, + "nauc_precision_at_20_max": 0.330971, + "nauc_precision_at_20_std": 0.037248, + "nauc_precision_at_20_diff1": 0.265846, + "nauc_precision_at_100_max": 0.226056, + "nauc_precision_at_100_std": 0.057313, + "nauc_precision_at_100_diff1": 0.118118, + "nauc_precision_at_1000_max": 0.022151, + "nauc_precision_at_1000_std": -0.008476, + "nauc_precision_at_1000_diff1": -0.115204, + "nauc_mrr_at_1_max": 0.370234, + "nauc_mrr_at_1_std": 0.01079, + "nauc_mrr_at_1_diff1": 0.588242, + "nauc_mrr_at_3_max": 0.37746, + "nauc_mrr_at_3_std": -0.001059, + "nauc_mrr_at_3_diff1": 0.554296, + "nauc_mrr_at_5_max": 0.37591, + "nauc_mrr_at_5_std": -0.003099, + "nauc_mrr_at_5_diff1": 0.548899, + "nauc_mrr_at_10_max": 0.38009, + "nauc_mrr_at_10_std": -0.001757, + "nauc_mrr_at_10_diff1": 0.548653, + "nauc_mrr_at_20_max": 0.378978, + "nauc_mrr_at_20_std": -0.000594, + "nauc_mrr_at_20_diff1": 0.547794, + "nauc_mrr_at_100_max": 0.378925, + "nauc_mrr_at_100_std": -0.000104, + "nauc_mrr_at_100_diff1": 0.54878, + "nauc_mrr_at_1000_max": 0.378838, + "nauc_mrr_at_1000_std": -0.000258, + "nauc_mrr_at_1000_diff1": 0.548967, + "hit_rate_at_1": 0.39958, + "hit_rate_at_3": 0.55463, + "hit_rate_at_5": 0.61811, + "hit_rate_at_10": 0.69719, + "hit_rate_at_20": 0.76587, + "hit_rate_at_100": 0.89386, + "hit_rate_at_1000": 1.0, + "main_score": 0.53136, + "hf_subset": "sw", + "languages": [ + "swa-Latn" + ] + }, + { + "ndcg_at_1": 0.67222, + "ndcg_at_3": 0.75249, + "ndcg_at_5": 0.77696, + "ndcg_at_10": 0.79392, + "ndcg_at_20": 0.80334, + "ndcg_at_100": 0.8101, + "ndcg_at_1000": 0.81134, + "map_at_1": 0.65296, + "map_at_3": 0.73, + "map_at_5": 0.74428, + "map_at_10": 0.75157, + "map_at_20": 0.75431, + "map_at_100": 0.75542, + "map_at_1000": 0.75548, + "recall_at_1": 0.65296, + "recall_at_3": 0.81167, + "recall_at_5": 0.86944, + "recall_at_10": 0.92056, + "recall_at_20": 0.95704, + "recall_at_100": 0.99111, + "recall_at_1000": 1.0, + "accuracy": 0.65296, + "precision_at_1": 0.67222, + "precision_at_3": 0.28333, + "precision_at_5": 0.18222, + "precision_at_10": 0.09678, + "precision_at_20": 0.05033, + "precision_at_100": 0.01043, + "precision_at_1000": 0.00105, + "mrr_at_1": 0.672222, + "mrr_at_3": 0.736111, + "mrr_at_5": 0.749167, + "mrr_at_10": 0.755871, + "mrr_at_20": 0.758296, + "mrr_at_100": 0.759298, + "mrr_at_1000": 0.759347, + "nauc_ndcg_at_1_max": 0.287561, + "nauc_ndcg_at_1_std": -0.316923, + "nauc_ndcg_at_1_diff1": 0.756589, + "nauc_ndcg_at_3_max": 0.347555, + "nauc_ndcg_at_3_std": -0.363135, + "nauc_ndcg_at_3_diff1": 0.707958, + "nauc_ndcg_at_5_max": 0.339133, + "nauc_ndcg_at_5_std": -0.375813, + "nauc_ndcg_at_5_diff1": 0.708085, + "nauc_ndcg_at_10_max": 0.337983, + "nauc_ndcg_at_10_std": -0.360369, + "nauc_ndcg_at_10_diff1": 0.710961, + "nauc_ndcg_at_20_max": 0.346406, + "nauc_ndcg_at_20_std": -0.341529, + "nauc_ndcg_at_20_diff1": 0.717846, + "nauc_ndcg_at_100_max": 0.340218, + "nauc_ndcg_at_100_std": -0.339439, + "nauc_ndcg_at_100_diff1": 0.721039, + "nauc_ndcg_at_1000_max": 0.336201, + "nauc_ndcg_at_1000_std": -0.344408, + "nauc_ndcg_at_1000_diff1": 0.720776, + "nauc_map_at_1_max": 0.264602, + "nauc_map_at_1_std": -0.30969, + "nauc_map_at_1_diff1": 0.750731, + "nauc_map_at_3_max": 0.331752, + "nauc_map_at_3_std": -0.349923, + "nauc_map_at_3_diff1": 0.721481, + "nauc_map_at_5_max": 0.326204, + "nauc_map_at_5_std": -0.357416, + "nauc_map_at_5_diff1": 0.722245, + "nauc_map_at_10_max": 0.326751, + "nauc_map_at_10_std": -0.350449, + "nauc_map_at_10_diff1": 0.723723, + "nauc_map_at_20_max": 0.328329, + "nauc_map_at_20_std": -0.345787, + "nauc_map_at_20_diff1": 0.725224, + "nauc_map_at_100_max": 0.327674, + "nauc_map_at_100_std": -0.34522, + "nauc_map_at_100_diff1": 0.725543, + "nauc_map_at_1000_max": 0.327517, + "nauc_map_at_1000_std": -0.34539, + "nauc_map_at_1000_diff1": 0.725529, + "nauc_recall_at_1_max": 0.264602, + "nauc_recall_at_1_std": -0.30969, + "nauc_recall_at_1_diff1": 0.750731, + "nauc_recall_at_3_max": 0.392903, + "nauc_recall_at_3_std": -0.413312, + "nauc_recall_at_3_diff1": 0.652703, + "nauc_recall_at_5_max": 0.383876, + "nauc_recall_at_5_std": -0.477044, + "nauc_recall_at_5_diff1": 0.63391, + "nauc_recall_at_10_max": 0.394488, + "nauc_recall_at_10_std": -0.438082, + "nauc_recall_at_10_diff1": 0.611705, + "nauc_recall_at_20_max": 0.580669, + "nauc_recall_at_20_std": -0.21724, + "nauc_recall_at_20_diff1": 0.64524, + "nauc_recall_at_100_max": 0.871178, + "nauc_recall_at_100_std": 0.335726, + "nauc_recall_at_100_diff1": 0.750088, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.287561, + "nauc_precision_at_1_std": -0.316923, + "nauc_precision_at_1_diff1": 0.756589, + "nauc_precision_at_3_max": 0.336497, + "nauc_precision_at_3_std": -0.322416, + "nauc_precision_at_3_diff1": 0.448388, + "nauc_precision_at_5_max": 0.287744, + "nauc_precision_at_5_std": -0.331085, + "nauc_precision_at_5_diff1": 0.342789, + "nauc_precision_at_10_max": 0.235391, + "nauc_precision_at_10_std": -0.21488, + "nauc_precision_at_10_diff1": 0.18874, + "nauc_precision_at_20_max": 0.227631, + "nauc_precision_at_20_std": -0.03525, + "nauc_precision_at_20_diff1": 0.046406, + "nauc_precision_at_100_max": 0.111202, + "nauc_precision_at_100_std": 0.089929, + "nauc_precision_at_100_diff1": -0.174913, + "nauc_precision_at_1000_max": 0.02213, + "nauc_precision_at_1000_std": 0.0561, + "nauc_precision_at_1000_diff1": -0.263662, + "nauc_mrr_at_1_max": 0.287561, + "nauc_mrr_at_1_std": -0.316923, + "nauc_mrr_at_1_diff1": 0.756589, + "nauc_mrr_at_3_max": 0.340127, + "nauc_mrr_at_3_std": -0.346521, + "nauc_mrr_at_3_diff1": 0.721422, + "nauc_mrr_at_5_max": 0.335323, + "nauc_mrr_at_5_std": -0.349213, + "nauc_mrr_at_5_diff1": 0.722392, + "nauc_mrr_at_10_max": 0.33354, + "nauc_mrr_at_10_std": -0.34369, + "nauc_mrr_at_10_diff1": 0.723638, + "nauc_mrr_at_20_max": 0.335433, + "nauc_mrr_at_20_std": -0.339886, + "nauc_mrr_at_20_diff1": 0.725369, + "nauc_mrr_at_100_max": 0.334479, + "nauc_mrr_at_100_std": -0.339365, + "nauc_mrr_at_100_diff1": 0.725754, + "nauc_mrr_at_1000_max": 0.334353, + "nauc_mrr_at_1000_std": -0.339489, + "nauc_mrr_at_1000_diff1": 0.725744, + "hit_rate_at_1": 0.67222, + "hit_rate_at_3": 0.81889, + "hit_rate_at_5": 0.87444, + "hit_rate_at_10": 0.92444, + "hit_rate_at_20": 0.95889, + "hit_rate_at_100": 0.99222, + "hit_rate_at_1000": 1.0, + "main_score": 0.79392, + "hf_subset": "hi", + "languages": [ + "hin-Deva" + ] + }, + { + "ndcg_at_1": 0.59653, + "ndcg_at_3": 0.68849, + "ndcg_at_5": 0.71158, + "ndcg_at_10": 0.73366, + "ndcg_at_20": 0.74549, + "ndcg_at_100": 0.75765, + "ndcg_at_1000": 0.76046, + "map_at_1": 0.57201, + "map_at_3": 0.65912, + "map_at_5": 0.67384, + "map_at_10": 0.68382, + "map_at_20": 0.68748, + "map_at_100": 0.68935, + "map_at_1000": 0.6895, + "recall_at_1": 0.57201, + "recall_at_3": 0.75468, + "recall_at_5": 0.80712, + "recall_at_10": 0.873, + "recall_at_20": 0.9176, + "recall_at_100": 0.98008, + "recall_at_1000": 1.0, + "accuracy": 0.57201, + "precision_at_1": 0.59653, + "precision_at_3": 0.26762, + "precision_at_5": 0.17406, + "precision_at_10": 0.09448, + "precision_at_20": 0.0499, + "precision_at_100": 0.01067, + "precision_at_1000": 0.00109, + "mrr_at_1": 0.596527, + "mrr_at_3": 0.678413, + "mrr_at_5": 0.688781, + "mrr_at_10": 0.69684, + "mrr_at_20": 0.699632, + "mrr_at_100": 0.7013, + "mrr_at_1000": 0.701408, + "nauc_ndcg_at_1_max": 0.330913, + "nauc_ndcg_at_1_std": -0.190621, + "nauc_ndcg_at_1_diff1": 0.670737, + "nauc_ndcg_at_3_max": 0.382742, + "nauc_ndcg_at_3_std": -0.223201, + "nauc_ndcg_at_3_diff1": 0.636972, + "nauc_ndcg_at_5_max": 0.398319, + "nauc_ndcg_at_5_std": -0.224641, + "nauc_ndcg_at_5_diff1": 0.627676, + "nauc_ndcg_at_10_max": 0.395579, + "nauc_ndcg_at_10_std": -0.22406, + "nauc_ndcg_at_10_diff1": 0.624612, + "nauc_ndcg_at_20_max": 0.395167, + "nauc_ndcg_at_20_std": -0.208853, + "nauc_ndcg_at_20_diff1": 0.629325, + "nauc_ndcg_at_100_max": 0.389046, + "nauc_ndcg_at_100_std": -0.209605, + "nauc_ndcg_at_100_diff1": 0.635886, + "nauc_ndcg_at_1000_max": 0.385539, + "nauc_ndcg_at_1000_std": -0.210165, + "nauc_ndcg_at_1000_diff1": 0.637387, + "nauc_map_at_1_max": 0.3117, + "nauc_map_at_1_std": -0.189162, + "nauc_map_at_1_diff1": 0.672295, + "nauc_map_at_3_max": 0.366693, + "nauc_map_at_3_std": -0.216074, + "nauc_map_at_3_diff1": 0.646438, + "nauc_map_at_5_max": 0.376806, + "nauc_map_at_5_std": -0.215562, + "nauc_map_at_5_diff1": 0.642306, + "nauc_map_at_10_max": 0.37659, + "nauc_map_at_10_std": -0.214531, + "nauc_map_at_10_diff1": 0.641372, + "nauc_map_at_20_max": 0.376534, + "nauc_map_at_20_std": -0.210248, + "nauc_map_at_20_diff1": 0.642649, + "nauc_map_at_100_max": 0.375666, + "nauc_map_at_100_std": -0.210245, + "nauc_map_at_100_diff1": 0.643409, + "nauc_map_at_1000_max": 0.375509, + "nauc_map_at_1000_std": -0.210289, + "nauc_map_at_1000_diff1": 0.643476, + "nauc_recall_at_1_max": 0.3117, + "nauc_recall_at_1_std": -0.189162, + "nauc_recall_at_1_diff1": 0.672295, + "nauc_recall_at_3_max": 0.418695, + "nauc_recall_at_3_std": -0.252592, + "nauc_recall_at_3_diff1": 0.592025, + "nauc_recall_at_5_max": 0.470079, + "nauc_recall_at_5_std": -0.267159, + "nauc_recall_at_5_diff1": 0.556567, + "nauc_recall_at_10_max": 0.489755, + "nauc_recall_at_10_std": -0.285282, + "nauc_recall_at_10_diff1": 0.514702, + "nauc_recall_at_20_max": 0.531844, + "nauc_recall_at_20_std": -0.178128, + "nauc_recall_at_20_diff1": 0.512994, + "nauc_recall_at_100_max": 0.63168, + "nauc_recall_at_100_std": -0.161234, + "nauc_recall_at_100_diff1": 0.523609, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.330913, + "nauc_precision_at_1_std": -0.190621, + "nauc_precision_at_1_diff1": 0.670737, + "nauc_precision_at_3_max": 0.421426, + "nauc_precision_at_3_std": -0.190041, + "nauc_precision_at_3_diff1": 0.465348, + "nauc_precision_at_5_max": 0.402524, + "nauc_precision_at_5_std": -0.143462, + "nauc_precision_at_5_diff1": 0.316406, + "nauc_precision_at_10_max": 0.344014, + "nauc_precision_at_10_std": -0.088411, + "nauc_precision_at_10_diff1": 0.166348, + "nauc_precision_at_20_max": 0.279966, + "nauc_precision_at_20_std": 0.034083, + "nauc_precision_at_20_diff1": 0.05643, + "nauc_precision_at_100_max": 0.157264, + "nauc_precision_at_100_std": 0.132156, + "nauc_precision_at_100_diff1": -0.143941, + "nauc_precision_at_1000_max": 0.070566, + "nauc_precision_at_1000_std": 0.162961, + "nauc_precision_at_1000_diff1": -0.222775, + "nauc_mrr_at_1_max": 0.330913, + "nauc_mrr_at_1_std": -0.190621, + "nauc_mrr_at_1_diff1": 0.670737, + "nauc_mrr_at_3_max": 0.375168, + "nauc_mrr_at_3_std": -0.211379, + "nauc_mrr_at_3_diff1": 0.645568, + "nauc_mrr_at_5_max": 0.379154, + "nauc_mrr_at_5_std": -0.213605, + "nauc_mrr_at_5_diff1": 0.64039, + "nauc_mrr_at_10_max": 0.375028, + "nauc_mrr_at_10_std": -0.214501, + "nauc_mrr_at_10_diff1": 0.640499, + "nauc_mrr_at_20_max": 0.374777, + "nauc_mrr_at_20_std": -0.211247, + "nauc_mrr_at_20_diff1": 0.641949, + "nauc_mrr_at_100_max": 0.373881, + "nauc_mrr_at_100_std": -0.211248, + "nauc_mrr_at_100_diff1": 0.642682, + "nauc_mrr_at_1000_max": 0.373771, + "nauc_mrr_at_1000_std": -0.211248, + "nauc_mrr_at_1000_diff1": 0.642732, + "hit_rate_at_1": 0.59653, + "hit_rate_at_3": 0.77835, + "hit_rate_at_5": 0.82329, + "hit_rate_at_10": 0.88355, + "hit_rate_at_20": 0.92339, + "hit_rate_at_100": 0.98264, + "hit_rate_at_1000": 1.0, + "main_score": 0.73366, + "hf_subset": "ur", + "languages": [ + "urd-Arab" + ] + } + ] + }, + "evaluation_time": 5900.7284371852875, + "kg_co2_emissions": null, + "date": 1777928312.845543 +} \ No newline at end of file diff --git a/results/Qwen__Qwen3-Embedding-0.6B/b22da495047858cce924d27d76261e96be6febc0/LexRetrieval.v1.json b/results/Qwen__Qwen3-Embedding-0.6B/b22da495047858cce924d27d76261e96be6febc0/LexRetrieval.v1.json new file mode 100644 index 0000000000..d2f8678bfa --- /dev/null +++ b/results/Qwen__Qwen3-Embedding-0.6B/b22da495047858cce924d27d76261e96be6febc0/LexRetrieval.v1.json @@ -0,0 +1,167 @@ +{ + "dataset_revision": "b5b22dee3a30bb32ac579c14afe9eb152e8c70c7", + "task_name": "LexRetrieval.v1", + "mteb_version": "2.12.38", + "scores": { + "test": [ + { + "ndcg_at_1": 0.91609, + "ndcg_at_3": 0.94081, + "ndcg_at_5": 0.94435, + "ndcg_at_10": 0.94641, + "ndcg_at_20": 0.9477, + "ndcg_at_100": 0.95007, + "ndcg_at_1000": 0.95101, + "map_at_1": 0.91609, + "map_at_3": 0.9352, + "map_at_5": 0.93717, + "map_at_10": 0.93802, + "map_at_20": 0.93838, + "map_at_100": 0.93872, + "map_at_1000": 0.93875, + "recall_at_1": 0.91609, + "recall_at_3": 0.95687, + "recall_at_5": 0.96541, + "recall_at_10": 0.97182, + "recall_at_20": 0.97683, + "recall_at_100": 0.98954, + "recall_at_1000": 0.99701, + "accuracy": 0.91609, + "precision_at_1": 0.91609, + "precision_at_3": 0.31896, + "precision_at_5": 0.19308, + "precision_at_10": 0.09718, + "precision_at_20": 0.04884, + "precision_at_100": 0.0099, + "precision_at_1000": 0.001, + "mrr_at_1": 0.916088, + "mrr_at_3": 0.935198, + "mrr_at_5": 0.937168, + "mrr_at_10": 0.93802, + "mrr_at_20": 0.93838, + "mrr_at_100": 0.938715, + "mrr_at_1000": 0.938748, + "nauc_ndcg_at_1_max": 0.81673, + "nauc_ndcg_at_1_std": 0.247894, + "nauc_ndcg_at_1_diff1": 0.917852, + "nauc_ndcg_at_3_max": 0.876839, + "nauc_ndcg_at_3_std": 0.321955, + "nauc_ndcg_at_3_diff1": 0.926465, + "nauc_ndcg_at_5_max": 0.875659, + "nauc_ndcg_at_5_std": 0.32546, + "nauc_ndcg_at_5_diff1": 0.927713, + "nauc_ndcg_at_10_max": 0.873106, + "nauc_ndcg_at_10_std": 0.323085, + "nauc_ndcg_at_10_diff1": 0.926833, + "nauc_ndcg_at_20_max": 0.870988, + "nauc_ndcg_at_20_std": 0.32683, + "nauc_ndcg_at_20_diff1": 0.926007, + "nauc_ndcg_at_100_max": 0.867433, + "nauc_ndcg_at_100_std": 0.322516, + "nauc_ndcg_at_100_diff1": 0.925425, + "nauc_ndcg_at_1000_max": 0.865375, + "nauc_ndcg_at_1000_std": 0.31797, + "nauc_ndcg_at_1000_diff1": 0.925144, + "nauc_map_at_1_max": 0.81673, + "nauc_map_at_1_std": 0.247894, + "nauc_map_at_1_diff1": 0.917852, + "nauc_map_at_3_max": 0.860706, + "nauc_map_at_3_std": 0.300296, + "nauc_map_at_3_diff1": 0.923911, + "nauc_map_at_5_max": 0.859547, + "nauc_map_at_5_std": 0.301341, + "nauc_map_at_5_diff1": 0.924352, + "nauc_map_at_10_max": 0.858437, + "nauc_map_at_10_std": 0.300227, + "nauc_map_at_10_diff1": 0.924005, + "nauc_map_at_20_max": 0.857845, + "nauc_map_at_20_std": 0.300986, + "nauc_map_at_20_diff1": 0.923791, + "nauc_map_at_100_max": 0.857383, + "nauc_map_at_100_std": 0.300307, + "nauc_map_at_100_diff1": 0.92374, + "nauc_map_at_1000_max": 0.857323, + "nauc_map_at_1000_std": 0.300206, + "nauc_map_at_1000_diff1": 0.923733, + "nauc_recall_at_1_max": 0.81673, + "nauc_recall_at_1_std": 0.247894, + "nauc_recall_at_1_diff1": 0.917852, + "nauc_recall_at_3_max": 0.945151, + "nauc_recall_at_3_std": 0.414952, + "nauc_recall_at_3_diff1": 0.937451, + "nauc_recall_at_5_max": 0.957958, + "nauc_recall_at_5_std": 0.451598, + "nauc_recall_at_5_diff1": 0.945736, + "nauc_recall_at_10_max": 0.961381, + "nauc_recall_at_10_std": 0.465541, + "nauc_recall_at_10_diff1": 0.9445, + "nauc_recall_at_20_max": 0.961848, + "nauc_recall_at_20_std": 0.529496, + "nauc_recall_at_20_diff1": 0.941064, + "nauc_recall_at_100_max": 0.979825, + "nauc_recall_at_100_std": 0.669628, + "nauc_recall_at_100_diff1": 0.942167, + "nauc_recall_at_1000_max": 0.990663, + "nauc_recall_at_1000_std": 0.90938, + "nauc_recall_at_1000_diff1": 0.945561, + "nauc_precision_at_1_max": 0.81673, + "nauc_precision_at_1_std": 0.247894, + "nauc_precision_at_1_diff1": 0.917852, + "nauc_precision_at_3_max": 0.945151, + "nauc_precision_at_3_std": 0.414952, + "nauc_precision_at_3_diff1": 0.937451, + "nauc_precision_at_5_max": 0.957958, + "nauc_precision_at_5_std": 0.451598, + "nauc_precision_at_5_diff1": 0.945736, + "nauc_precision_at_10_max": 0.961381, + "nauc_precision_at_10_std": 0.465541, + "nauc_precision_at_10_diff1": 0.9445, + "nauc_precision_at_20_max": 0.961848, + "nauc_precision_at_20_std": 0.529496, + "nauc_precision_at_20_diff1": 0.941064, + "nauc_precision_at_100_max": 0.979825, + "nauc_precision_at_100_std": 0.669628, + "nauc_precision_at_100_diff1": 0.942167, + "nauc_precision_at_1000_max": 0.990663, + "nauc_precision_at_1000_std": 0.90938, + "nauc_precision_at_1000_diff1": 0.945561, + "nauc_mrr_at_1_max": 0.81673, + "nauc_mrr_at_1_std": 0.247894, + "nauc_mrr_at_1_diff1": 0.917852, + "nauc_mrr_at_3_max": 0.860706, + "nauc_mrr_at_3_std": 0.300296, + "nauc_mrr_at_3_diff1": 0.923911, + "nauc_mrr_at_5_max": 0.859547, + "nauc_mrr_at_5_std": 0.301341, + "nauc_mrr_at_5_diff1": 0.924352, + "nauc_mrr_at_10_max": 0.858437, + "nauc_mrr_at_10_std": 0.300227, + "nauc_mrr_at_10_diff1": 0.924005, + "nauc_mrr_at_20_max": 0.857845, + "nauc_mrr_at_20_std": 0.300986, + "nauc_mrr_at_20_diff1": 0.923791, + "nauc_mrr_at_100_max": 0.857383, + "nauc_mrr_at_100_std": 0.300307, + "nauc_mrr_at_100_diff1": 0.92374, + "nauc_mrr_at_1000_max": 0.857323, + "nauc_mrr_at_1000_std": 0.300206, + "nauc_mrr_at_1000_diff1": 0.923733, + "hit_rate_at_1": 0.91609, + "hit_rate_at_3": 0.95687, + "hit_rate_at_5": 0.96541, + "hit_rate_at_10": 0.97182, + "hit_rate_at_20": 0.97683, + "hit_rate_at_100": 0.98954, + "hit_rate_at_1000": 0.99701, + "main_score": 0.94641, + "hf_subset": "default", + "languages": [ + "dan-Latn" + ] + } + ] + }, + "evaluation_time": 4798.954032421112, + "kg_co2_emissions": null, + "date": 1777922394.070889 +} \ No newline at end of file diff --git a/results/Qwen__Qwen3-Embedding-0.6B/b22da495047858cce924d27d76261e96be6febc0/PublicNewsRetrieval.json b/results/Qwen__Qwen3-Embedding-0.6B/b22da495047858cce924d27d76261e96be6febc0/PublicNewsRetrieval.json new file mode 100644 index 0000000000..eb02d3520c --- /dev/null +++ b/results/Qwen__Qwen3-Embedding-0.6B/b22da495047858cce924d27d76261e96be6febc0/PublicNewsRetrieval.json @@ -0,0 +1,322 @@ +{ + "dataset_revision": "82dad8c469646feae9631d69ba92bebfc0a586e9", + "task_name": "PublicNewsRetrieval", + "mteb_version": "2.12.37", + "scores": { + "test": [ + { + "ndcg_at_1": 0.69169, + "ndcg_at_3": 0.74337, + "ndcg_at_5": 0.75868, + "ndcg_at_10": 0.77239, + "ndcg_at_20": 0.7839, + "ndcg_at_100": 0.79556, + "ndcg_at_1000": 0.80063, + "map_at_1": 0.68404, + "map_at_3": 0.73023, + "map_at_5": 0.73918, + "map_at_10": 0.74498, + "map_at_20": 0.74837, + "map_at_100": 0.75011, + "map_at_1000": 0.75031, + "recall_at_1": 0.68404, + "recall_at_3": 0.7767, + "recall_at_5": 0.81311, + "recall_at_10": 0.85422, + "recall_at_20": 0.89818, + "recall_at_100": 0.95856, + "recall_at_1000": 0.99726, + "accuracy": 0.68404, + "precision_at_1": 0.69169, + "precision_at_3": 0.26475, + "precision_at_5": 0.16716, + "precision_at_10": 0.08834, + "precision_at_20": 0.04662, + "precision_at_100": 0.00999, + "precision_at_1000": 0.00104, + "mrr_at_1": 0.691689, + "mrr_at_3": 0.734138, + "mrr_at_5": 0.742181, + "mrr_at_10": 0.747777, + "mrr_at_20": 0.750824, + "mrr_at_100": 0.752474, + "mrr_at_1000": 0.752647, + "nauc_ndcg_at_1_max": 0.530176, + "nauc_ndcg_at_1_std": -0.354692, + "nauc_ndcg_at_1_diff1": 0.867239, + "nauc_ndcg_at_3_max": 0.536768, + "nauc_ndcg_at_3_std": -0.376353, + "nauc_ndcg_at_3_diff1": 0.845334, + "nauc_ndcg_at_5_max": 0.544557, + "nauc_ndcg_at_5_std": -0.378648, + "nauc_ndcg_at_5_diff1": 0.842731, + "nauc_ndcg_at_10_max": 0.559433, + "nauc_ndcg_at_10_std": -0.360403, + "nauc_ndcg_at_10_diff1": 0.844238, + "nauc_ndcg_at_20_max": 0.569563, + "nauc_ndcg_at_20_std": -0.342336, + "nauc_ndcg_at_20_diff1": 0.840343, + "nauc_ndcg_at_100_max": 0.562957, + "nauc_ndcg_at_100_std": -0.344107, + "nauc_ndcg_at_100_diff1": 0.844738, + "nauc_ndcg_at_1000_max": 0.557923, + "nauc_ndcg_at_1000_std": -0.351978, + "nauc_ndcg_at_1000_diff1": 0.846573, + "nauc_map_at_1_max": 0.512792, + "nauc_map_at_1_std": -0.351699, + "nauc_map_at_1_diff1": 0.866709, + "nauc_map_at_3_max": 0.533712, + "nauc_map_at_3_std": -0.369838, + "nauc_map_at_3_diff1": 0.850917, + "nauc_map_at_5_max": 0.538681, + "nauc_map_at_5_std": -0.371483, + "nauc_map_at_5_diff1": 0.84956, + "nauc_map_at_10_max": 0.545004, + "nauc_map_at_10_std": -0.364418, + "nauc_map_at_10_diff1": 0.850484, + "nauc_map_at_20_max": 0.548091, + "nauc_map_at_20_std": -0.359877, + "nauc_map_at_20_diff1": 0.849567, + "nauc_map_at_100_max": 0.547287, + "nauc_map_at_100_std": -0.360043, + "nauc_map_at_100_diff1": 0.85021, + "nauc_map_at_1000_max": 0.547117, + "nauc_map_at_1000_std": -0.36025, + "nauc_map_at_1000_diff1": 0.850254, + "nauc_recall_at_1_max": 0.512792, + "nauc_recall_at_1_std": -0.351699, + "nauc_recall_at_1_diff1": 0.866709, + "nauc_recall_at_3_max": 0.536337, + "nauc_recall_at_3_std": -0.393533, + "nauc_recall_at_3_diff1": 0.824997, + "nauc_recall_at_5_max": 0.557214, + "nauc_recall_at_5_std": -0.404194, + "nauc_recall_at_5_diff1": 0.813933, + "nauc_recall_at_10_max": 0.620287, + "nauc_recall_at_10_std": -0.331356, + "nauc_recall_at_10_diff1": 0.813303, + "nauc_recall_at_20_max": 0.708767, + "nauc_recall_at_20_std": -0.187852, + "nauc_recall_at_20_diff1": 0.773897, + "nauc_recall_at_100_max": 0.744852, + "nauc_recall_at_100_std": -0.020637, + "nauc_recall_at_100_diff1": 0.768399, + "nauc_recall_at_1000_max": 0.854012, + "nauc_recall_at_1000_std": 0.33573, + "nauc_recall_at_1000_diff1": 0.786001, + "nauc_precision_at_1_max": 0.530176, + "nauc_precision_at_1_std": -0.354692, + "nauc_precision_at_1_diff1": 0.867239, + "nauc_precision_at_3_max": 0.525273, + "nauc_precision_at_3_std": -0.349082, + "nauc_precision_at_3_diff1": 0.719997, + "nauc_precision_at_5_max": 0.517095, + "nauc_precision_at_5_std": -0.319314, + "nauc_precision_at_5_diff1": 0.641428, + "nauc_precision_at_10_max": 0.549133, + "nauc_precision_at_10_std": -0.202145, + "nauc_precision_at_10_diff1": 0.551139, + "nauc_precision_at_20_max": 0.559674, + "nauc_precision_at_20_std": -0.022391, + "nauc_precision_at_20_diff1": 0.381004, + "nauc_precision_at_100_max": 0.389686, + "nauc_precision_at_100_std": 0.181317, + "nauc_precision_at_100_diff1": 0.076441, + "nauc_precision_at_1000_max": 0.093994, + "nauc_precision_at_1000_std": 0.252268, + "nauc_precision_at_1000_diff1": -0.31938, + "nauc_mrr_at_1_max": 0.530176, + "nauc_mrr_at_1_std": -0.354692, + "nauc_mrr_at_1_diff1": 0.867239, + "nauc_mrr_at_3_max": 0.539973, + "nauc_mrr_at_3_std": -0.372708, + "nauc_mrr_at_3_diff1": 0.851126, + "nauc_mrr_at_5_max": 0.543341, + "nauc_mrr_at_5_std": -0.372616, + "nauc_mrr_at_5_diff1": 0.850062, + "nauc_mrr_at_10_max": 0.547957, + "nauc_mrr_at_10_std": -0.366278, + "nauc_mrr_at_10_diff1": 0.850837, + "nauc_mrr_at_20_max": 0.549343, + "nauc_mrr_at_20_std": -0.362884, + "nauc_mrr_at_20_diff1": 0.850114, + "nauc_mrr_at_100_max": 0.54857, + "nauc_mrr_at_100_std": -0.363222, + "nauc_mrr_at_100_diff1": 0.850675, + "nauc_mrr_at_1000_max": 0.548442, + "nauc_mrr_at_1000_std": -0.36338, + "nauc_mrr_at_1000_diff1": 0.850728, + "hit_rate_at_1": 0.69169, + "hit_rate_at_3": 0.78217, + "hit_rate_at_5": 0.81769, + "hit_rate_at_10": 0.85992, + "hit_rate_at_20": 0.90214, + "hit_rate_at_100": 0.96247, + "hit_rate_at_1000": 0.99799, + "main_score": 0.77239, + "hf_subset": "pl", + "languages": [ + "pol-Latn" + ] + }, + { + "ndcg_at_1": 0.62459, + "ndcg_at_3": 0.70401, + "ndcg_at_5": 0.72703, + "ndcg_at_10": 0.7465, + "ndcg_at_20": 0.7562, + "ndcg_at_100": 0.76967, + "ndcg_at_1000": 0.7725, + "map_at_1": 0.60717, + "map_at_3": 0.68107, + "map_at_5": 0.69449, + "map_at_10": 0.70326, + "map_at_20": 0.70606, + "map_at_100": 0.7081, + "map_at_1000": 0.70824, + "recall_at_1": 0.60717, + "recall_at_3": 0.76052, + "recall_at_5": 0.81521, + "recall_at_10": 0.87289, + "recall_at_20": 0.91017, + "recall_at_100": 0.97983, + "recall_at_1000": 0.99976, + "accuracy": 0.60717, + "precision_at_1": 0.62459, + "precision_at_3": 0.26572, + "precision_at_5": 0.17163, + "precision_at_10": 0.09253, + "precision_at_20": 0.04835, + "precision_at_100": 0.01045, + "precision_at_1000": 0.00107, + "mrr_at_1": 0.624586, + "mrr_at_3": 0.690701, + "mrr_at_5": 0.702711, + "mrr_at_10": 0.710035, + "mrr_at_20": 0.712566, + "mrr_at_100": 0.714385, + "mrr_at_1000": 0.714492, + "nauc_ndcg_at_1_max": 0.293095, + "nauc_ndcg_at_1_std": -0.436165, + "nauc_ndcg_at_1_diff1": 0.793064, + "nauc_ndcg_at_3_max": 0.306647, + "nauc_ndcg_at_3_std": -0.49847, + "nauc_ndcg_at_3_diff1": 0.756482, + "nauc_ndcg_at_5_max": 0.314827, + "nauc_ndcg_at_5_std": -0.503434, + "nauc_ndcg_at_5_diff1": 0.755706, + "nauc_ndcg_at_10_max": 0.313949, + "nauc_ndcg_at_10_std": -0.513259, + "nauc_ndcg_at_10_diff1": 0.757052, + "nauc_ndcg_at_20_max": 0.310376, + "nauc_ndcg_at_20_std": -0.508891, + "nauc_ndcg_at_20_diff1": 0.758328, + "nauc_ndcg_at_100_max": 0.307814, + "nauc_ndcg_at_100_std": -0.498156, + "nauc_ndcg_at_100_diff1": 0.762996, + "nauc_ndcg_at_1000_max": 0.308432, + "nauc_ndcg_at_1000_std": -0.49328, + "nauc_ndcg_at_1000_diff1": 0.763743, + "nauc_map_at_1_max": 0.272139, + "nauc_map_at_1_std": -0.422872, + "nauc_map_at_1_diff1": 0.790452, + "nauc_map_at_3_max": 0.2991, + "nauc_map_at_3_std": -0.482079, + "nauc_map_at_3_diff1": 0.7646, + "nauc_map_at_5_max": 0.303084, + "nauc_map_at_5_std": -0.485508, + "nauc_map_at_5_diff1": 0.764591, + "nauc_map_at_10_max": 0.303746, + "nauc_map_at_10_std": -0.488607, + "nauc_map_at_10_diff1": 0.76547, + "nauc_map_at_20_max": 0.30281, + "nauc_map_at_20_std": -0.487098, + "nauc_map_at_20_diff1": 0.765762, + "nauc_map_at_100_max": 0.302583, + "nauc_map_at_100_std": -0.485812, + "nauc_map_at_100_diff1": 0.766386, + "nauc_map_at_1000_max": 0.302626, + "nauc_map_at_1000_std": -0.485527, + "nauc_map_at_1000_diff1": 0.766427, + "nauc_recall_at_1_max": 0.272139, + "nauc_recall_at_1_std": -0.422872, + "nauc_recall_at_1_diff1": 0.790452, + "nauc_recall_at_3_max": 0.320859, + "nauc_recall_at_3_std": -0.551108, + "nauc_recall_at_3_diff1": 0.720849, + "nauc_recall_at_5_max": 0.346939, + "nauc_recall_at_5_std": -0.576915, + "nauc_recall_at_5_diff1": 0.707802, + "nauc_recall_at_10_max": 0.352348, + "nauc_recall_at_10_std": -0.66198, + "nauc_recall_at_10_diff1": 0.700364, + "nauc_recall_at_20_max": 0.334316, + "nauc_recall_at_20_std": -0.685221, + "nauc_recall_at_20_diff1": 0.692932, + "nauc_recall_at_100_max": 0.272249, + "nauc_recall_at_100_std": -0.77754, + "nauc_recall_at_100_diff1": 0.713695, + "nauc_recall_at_1000_max": -0.171538, + "nauc_recall_at_1000_std": -0.564004, + "nauc_recall_at_1000_diff1": 0.722188, + "nauc_precision_at_1_max": 0.293095, + "nauc_precision_at_1_std": -0.436165, + "nauc_precision_at_1_diff1": 0.793064, + "nauc_precision_at_3_max": 0.318712, + "nauc_precision_at_3_std": -0.473126, + "nauc_precision_at_3_diff1": 0.561412, + "nauc_precision_at_5_max": 0.321733, + "nauc_precision_at_5_std": -0.443496, + "nauc_precision_at_5_diff1": 0.473932, + "nauc_precision_at_10_max": 0.277742, + "nauc_precision_at_10_std": -0.38711, + "nauc_precision_at_10_diff1": 0.321232, + "nauc_precision_at_20_max": 0.231537, + "nauc_precision_at_20_std": -0.305332, + "nauc_precision_at_20_diff1": 0.21063, + "nauc_precision_at_100_max": 0.113724, + "nauc_precision_at_100_std": 0.025862, + "nauc_precision_at_100_diff1": -0.127939, + "nauc_precision_at_1000_max": 0.068329, + "nauc_precision_at_1000_std": 0.204522, + "nauc_precision_at_1000_diff1": -0.275848, + "nauc_mrr_at_1_max": 0.293095, + "nauc_mrr_at_1_std": -0.436165, + "nauc_mrr_at_1_diff1": 0.793064, + "nauc_mrr_at_3_max": 0.307015, + "nauc_mrr_at_3_std": -0.483118, + "nauc_mrr_at_3_diff1": 0.768187, + "nauc_mrr_at_5_max": 0.312005, + "nauc_mrr_at_5_std": -0.482449, + "nauc_mrr_at_5_diff1": 0.767537, + "nauc_mrr_at_10_max": 0.309708, + "nauc_mrr_at_10_std": -0.486227, + "nauc_mrr_at_10_diff1": 0.768272, + "nauc_mrr_at_20_max": 0.308847, + "nauc_mrr_at_20_std": -0.484734, + "nauc_mrr_at_20_diff1": 0.768567, + "nauc_mrr_at_100_max": 0.308554, + "nauc_mrr_at_100_std": -0.483849, + "nauc_mrr_at_100_diff1": 0.769301, + "nauc_mrr_at_1000_max": 0.308572, + "nauc_mrr_at_1000_std": -0.483689, + "nauc_mrr_at_1000_diff1": 0.769323, + "hit_rate_at_1": 0.62459, + "hit_rate_at_3": 0.7721, + "hit_rate_at_5": 0.82506, + "hit_rate_at_10": 0.87896, + "hit_rate_at_20": 0.91537, + "hit_rate_at_100": 0.98251, + "hit_rate_at_1000": 1.0, + "main_score": 0.7465, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ] + } + ] + }, + "evaluation_time": 72.4613733291626, + "kg_co2_emissions": null, + "date": 1777965318.964934 +} \ No newline at end of file diff --git a/results/Qwen__Qwen3-Embedding-0.6B/b22da495047858cce924d27d76261e96be6febc0/model_meta.json b/results/Qwen__Qwen3-Embedding-0.6B/b22da495047858cce924d27d76261e96be6febc0/model_meta.json index 34bd0ecc65..9b92ecfdb6 100644 --- a/results/Qwen__Qwen3-Embedding-0.6B/b22da495047858cce924d27d76261e96be6febc0/model_meta.json +++ b/results/Qwen__Qwen3-Embedding-0.6B/b22da495047858cce924d27d76261e96be6febc0/model_meta.json @@ -1 +1,128 @@ -{"loader_kwargs": {}, "name": "Qwen/Qwen3-Embedding-0.6B", "revision": "b22da495047858cce924d27d76261e96be6febc0", "release_date": "2025-06-05", "languages": ["afr-Latn", "ara-Arab", "aze-Latn", "bel-Cyrl", "bul-Cyrl", "ben-Beng", "cat-Latn", "ceb-Latn", "ces-Latn", "cym-Latn", "dan-Latn", "deu-Latn", "ell-Grek", "eng-Latn", "spa-Latn", "est-Latn", "eus-Latn", "fas-Arab", "fin-Latn", "fra-Latn", "glg-Latn", "guj-Gujr", "heb-Hebr", "hin-Deva", "hrv-Latn", "hat-Latn", "hun-Latn", "hye-Armn", "ind-Latn", "isl-Latn", "ita-Latn", "jpn-Jpan", "jav-Latn", "kat-Geor", "kaz-Cyrl", "khm-Khmr", "kan-Knda", "kor-Hang", "kir-Cyrl", "lao-Laoo", "lit-Latn", "lav-Latn", "mkd-Cyrl", "mal-Mlym", "mon-Cyrl", "mar-Deva", "msa-Latn", "mya-Mymr", "nep-Deva", "nld-Latn", "nor-Latn", "nob-Latn", "nno-Latn", "pan-Guru", "pol-Latn", "por-Latn", "que-Latn", "ron-Latn", "rus-Cyrl", "sin-Sinh", "slk-Latn", "slv-Latn", "swa-Latn", "tam-Taml", "tel-Telu", "tha-Thai", "tgl-Latn", "tur-Latn", "ukr-Cyrl", "urd-Arab", "vie-Latn", "yor-Latn", "zho-Hans"], "n_parameters": 595776512, "n_active_parameters_override": null, "n_embedding_parameters": 155309056, "memory_usage_mb": 1136.0, "max_tokens": 32768.0, "embed_dim": 1024, "license": "apache-2.0", "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["Sentence Transformers", "PyTorch", "safetensors", "Transformers"], "reference": "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B", "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": ["T2Retrieval", "HotpotQA", "CMedQAv2-reranking", "MIRACLRetrieval", "MSMARCO", "MMarcoReranking", "DuRetrieval", "CodeSearchNet", "MrTidyRetrieval", "FEVER", "NQ"], "adapted_from": null, "superseded_by": null, "modalities": ["text"], "model_type": ["dense"], "citation": "@article{qwen3embedding,\n title={Qwen3 Embedding: Advancing Text Embedding and Reranking Through Foundation Models},\n author={Zhang, Yanzhao and Li, Mingxin and Long, Dingkun and Zhang, Xin and Lin, Huan and Yang, Baosong and Xie, Pengjun and Yang, An and Liu, Dayiheng and Lin, Junyang and Huang, Fei and Zhou, Jingren},\n journal={arXiv preprint arXiv:2506.05176},\n year={2025}\n}", "contacts": null, "loader": "q3e_instruct_loader", "is_cross_encoder": false} \ No newline at end of file +{ + "loader_kwargs": {}, + "name": "Qwen/Qwen3-Embedding-0.6B", + "revision": "b22da495047858cce924d27d76261e96be6febc0", + "release_date": "2025-06-05", + "languages": [ + "afr-Latn", + "ara-Arab", + "aze-Latn", + "bel-Cyrl", + "bul-Cyrl", + "ben-Beng", + "cat-Latn", + "ceb-Latn", + "ces-Latn", + "cym-Latn", + "dan-Latn", + "deu-Latn", + "ell-Grek", + "eng-Latn", + "spa-Latn", + "est-Latn", + "eus-Latn", + "fas-Arab", + "fin-Latn", + "fra-Latn", + "glg-Latn", + "guj-Gujr", + "heb-Hebr", + "hin-Deva", + "hrv-Latn", + "hat-Latn", + "hun-Latn", + "hye-Armn", + "ind-Latn", + "isl-Latn", + "ita-Latn", + "jpn-Jpan", + "jav-Latn", + "kat-Geor", + "kaz-Cyrl", + "khm-Khmr", + "kan-Knda", + "kor-Hang", + "kir-Cyrl", + "lao-Laoo", + "lit-Latn", + "lav-Latn", + "mkd-Cyrl", + "mal-Mlym", + "mon-Cyrl", + "mar-Deva", + "msa-Latn", + "mya-Mymr", + "nep-Deva", + "nld-Latn", + "nor-Latn", + "nob-Latn", + "nno-Latn", + "pan-Guru", + "pol-Latn", + "por-Latn", + "que-Latn", + "ron-Latn", + "rus-Cyrl", + "sin-Sinh", + "slk-Latn", + "slv-Latn", + "swa-Latn", + "tam-Taml", + "tel-Telu", + "tha-Thai", + "tgl-Latn", + "tur-Latn", + "ukr-Cyrl", + "urd-Arab", + "vie-Latn", + "yor-Latn", + "zho-Hans" + ], + "n_parameters": 595776512, + "n_active_parameters_override": null, + "n_embedding_parameters": 155309056, + "memory_usage_mb": 1136.0, + "max_tokens": 32768.0, + "embed_dim": 1024, + "license": "apache-2.0", + "open_weights": true, + "public_training_code": null, + "public_training_data": null, + "framework": [ + "Sentence Transformers", + "PyTorch", + "safetensors", + "Transformers" + ], + "reference": "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B", + "similarity_fn_name": "cosine", + "use_instructions": true, + "training_datasets": [ + "MSMARCO", + "FEVER", + "NQ", + "CodeSearchNet", + "DuRetrieval", + "T2Retrieval", + "CMedQAv2-reranking", + "MMarcoReranking", + "MIRACLRetrieval", + "HotpotQA", + "MrTidyRetrieval" + ], + "adapted_from": null, + "superseded_by": null, + "modalities": [ + "text" + ], + "model_type": [ + "dense" + ], + "citation": "@article{qwen3embedding,\n title={Qwen3 Embedding: Advancing Text Embedding and Reranking Through Foundation Models},\n author={Zhang, Yanzhao and Li, Mingxin and Long, Dingkun and Zhang, Xin and Lin, Huan and Yang, Baosong and Xie, Pengjun and Yang, An and Liu, Dayiheng and Lin, Junyang and Huang, Fei and Zhou, Jingren},\n journal={arXiv preprint arXiv:2506.05176},\n year={2025}\n}", + "contacts": null, + "experiment_kwargs": null, + "output_dtypes": null, + "extra_requirements_groups": null, + "loader": "q3e_instruct_loader", + "is_cross_encoder": false +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/ElasticKBRetrieval.json b/results/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/ElasticKBRetrieval.json new file mode 100644 index 0000000000..326a4a438e --- /dev/null +++ b/results/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/ElasticKBRetrieval.json @@ -0,0 +1,324 @@ +{ + "dataset_revision": "21bdf1e024bf7c9f46720017559ce2f8c6116507", + "task_name": "ElasticKBRetrieval", + "mteb_version": "2.12.38", + "scores": { + "synthetic_test": [ + { + "ndcg_at_1": 0.60106, + "ndcg_at_3": 0.50276, + "ndcg_at_5": 0.47244, + "ndcg_at_10": 0.4538, + "ndcg_at_20": 0.453, + "ndcg_at_100": 0.49841, + "ndcg_at_1000": 0.56785, + "map_at_1": 0.18921, + "map_at_3": 0.24451, + "map_at_5": 0.261, + "map_at_10": 0.28016, + "map_at_20": 0.2965, + "map_at_100": 0.31974, + "map_at_1000": 0.33038, + "recall_at_1": 0.18921, + "recall_at_3": 0.28084, + "recall_at_5": 0.31505, + "recall_at_10": 0.36719, + "recall_at_20": 0.42821, + "recall_at_100": 0.59421, + "recall_at_1000": 0.82826, + "accuracy": 0.18921, + "precision_at_1": 0.60106, + "precision_at_3": 0.38652, + "precision_at_5": 0.3, + "precision_at_10": 0.2133, + "precision_at_20": 0.1492, + "precision_at_100": 0.05931, + "precision_at_1000": 0.01178, + "mrr_at_1": 0.601064, + "mrr_at_3": 0.705674, + "mrr_at_5": 0.717642, + "mrr_at_10": 0.723964, + "mrr_at_20": 0.725382, + "mrr_at_100": 0.726111, + "mrr_at_1000": 0.726264, + "nauc_ndcg_at_1_max": 0.259188, + "nauc_ndcg_at_1_std": -0.117735, + "nauc_ndcg_at_1_diff1": 0.473755, + "nauc_ndcg_at_3_max": 0.205073, + "nauc_ndcg_at_3_std": 0.008023, + "nauc_ndcg_at_3_diff1": 0.28159, + "nauc_ndcg_at_5_max": 0.217226, + "nauc_ndcg_at_5_std": 0.041813, + "nauc_ndcg_at_5_diff1": 0.304406, + "nauc_ndcg_at_10_max": 0.191352, + "nauc_ndcg_at_10_std": 0.044597, + "nauc_ndcg_at_10_diff1": 0.365266, + "nauc_ndcg_at_20_max": 0.189346, + "nauc_ndcg_at_20_std": 0.059517, + "nauc_ndcg_at_20_diff1": 0.40139, + "nauc_ndcg_at_100_max": 0.247528, + "nauc_ndcg_at_100_std": 0.156148, + "nauc_ndcg_at_100_diff1": 0.378973, + "nauc_ndcg_at_1000_max": 0.256891, + "nauc_ndcg_at_1000_std": 0.173467, + "nauc_ndcg_at_1000_diff1": 0.355549, + "nauc_map_at_1_max": 0.13219, + "nauc_map_at_1_std": -0.059478, + "nauc_map_at_1_diff1": 0.51635, + "nauc_map_at_3_max": 0.126025, + "nauc_map_at_3_std": -0.064232, + "nauc_map_at_3_diff1": 0.424384, + "nauc_map_at_5_max": 0.145166, + "nauc_map_at_5_std": -0.044299, + "nauc_map_at_5_diff1": 0.414989, + "nauc_map_at_10_max": 0.157993, + "nauc_map_at_10_std": -0.02027, + "nauc_map_at_10_diff1": 0.419525, + "nauc_map_at_20_max": 0.169927, + "nauc_map_at_20_std": 0.009232, + "nauc_map_at_20_diff1": 0.415501, + "nauc_map_at_100_max": 0.191279, + "nauc_map_at_100_std": 0.062749, + "nauc_map_at_100_diff1": 0.400896, + "nauc_map_at_1000_max": 0.191204, + "nauc_map_at_1000_std": 0.073134, + "nauc_map_at_1000_diff1": 0.39554, + "nauc_recall_at_1_max": 0.13219, + "nauc_recall_at_1_std": -0.059478, + "nauc_recall_at_1_diff1": 0.51635, + "nauc_recall_at_3_max": 0.112663, + "nauc_recall_at_3_std": -0.066409, + "nauc_recall_at_3_diff1": 0.334334, + "nauc_recall_at_5_max": 0.135786, + "nauc_recall_at_5_std": -0.026369, + "nauc_recall_at_5_diff1": 0.321528, + "nauc_recall_at_10_max": 0.147308, + "nauc_recall_at_10_std": -0.000727, + "nauc_recall_at_10_diff1": 0.32467, + "nauc_recall_at_20_max": 0.141486, + "nauc_recall_at_20_std": 0.058186, + "nauc_recall_at_20_diff1": 0.313931, + "nauc_recall_at_100_max": 0.212234, + "nauc_recall_at_100_std": 0.293729, + "nauc_recall_at_100_diff1": 0.23379, + "nauc_recall_at_1000_max": 0.23759, + "nauc_recall_at_1000_std": 0.421159, + "nauc_recall_at_1000_diff1": 0.120021, + "nauc_precision_at_1_max": 0.259188, + "nauc_precision_at_1_std": -0.117735, + "nauc_precision_at_1_diff1": 0.473755, + "nauc_precision_at_3_max": 0.179157, + "nauc_precision_at_3_std": 0.084704, + "nauc_precision_at_3_diff1": 0.025358, + "nauc_precision_at_5_max": 0.192061, + "nauc_precision_at_5_std": 0.172119, + "nauc_precision_at_5_diff1": -0.065714, + "nauc_precision_at_10_max": 0.110924, + "nauc_precision_at_10_std": 0.216725, + "nauc_precision_at_10_diff1": -0.120088, + "nauc_precision_at_20_max": 0.077713, + "nauc_precision_at_20_std": 0.255674, + "nauc_precision_at_20_diff1": -0.181416, + "nauc_precision_at_100_max": 0.010682, + "nauc_precision_at_100_std": 0.230694, + "nauc_precision_at_100_diff1": -0.240871, + "nauc_precision_at_1000_max": -0.144016, + "nauc_precision_at_1000_std": 0.049952, + "nauc_precision_at_1000_diff1": -0.279792, + "nauc_mrr_at_1_max": 0.259188, + "nauc_mrr_at_1_std": -0.117735, + "nauc_mrr_at_1_diff1": 0.473755, + "nauc_mrr_at_3_max": 0.252125, + "nauc_mrr_at_3_std": -0.057453, + "nauc_mrr_at_3_diff1": 0.456194, + "nauc_mrr_at_5_max": 0.261898, + "nauc_mrr_at_5_std": -0.047856, + "nauc_mrr_at_5_diff1": 0.47281, + "nauc_mrr_at_10_max": 0.252049, + "nauc_mrr_at_10_std": -0.058036, + "nauc_mrr_at_10_diff1": 0.469354, + "nauc_mrr_at_20_max": 0.254521, + "nauc_mrr_at_20_std": -0.059167, + "nauc_mrr_at_20_diff1": 0.470943, + "nauc_mrr_at_100_max": 0.253371, + "nauc_mrr_at_100_std": -0.057722, + "nauc_mrr_at_100_diff1": 0.470359, + "nauc_mrr_at_1000_max": 0.253515, + "nauc_mrr_at_1000_std": -0.057788, + "nauc_mrr_at_1000_diff1": 0.470166, + "hit_rate_at_1": 0.60106, + "hit_rate_at_3": 0.83511, + "hit_rate_at_5": 0.8883, + "hit_rate_at_10": 0.93085, + "hit_rate_at_20": 0.95213, + "hit_rate_at_100": 0.9734, + "hit_rate_at_1000": 1.0, + "main_score": 0.4538, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ], + "real_chat_test": [ + { + "ndcg_at_1": 0.72845, + "ndcg_at_3": 0.65786, + "ndcg_at_5": 0.61032, + "ndcg_at_10": 0.56309, + "ndcg_at_20": 0.52112, + "ndcg_at_100": 0.47019, + "ndcg_at_1000": 0.54353, + "map_at_1": 0.06, + "map_at_3": 0.1005, + "map_at_5": 0.12114, + "map_at_10": 0.14387, + "map_at_20": 0.16302, + "map_at_100": 0.20641, + "map_at_1000": 0.25384, + "recall_at_1": 0.06, + "recall_at_3": 0.10715, + "recall_at_5": 0.13974, + "recall_at_10": 0.18395, + "recall_at_20": 0.22747, + "recall_at_100": 0.38054, + "recall_at_1000": 0.68711, + "accuracy": 0.06, + "precision_at_1": 0.72845, + "precision_at_3": 0.625, + "precision_at_5": 0.54828, + "precision_at_10": 0.46422, + "precision_at_20": 0.37759, + "precision_at_100": 0.21789, + "precision_at_1000": 0.0754, + "mrr_at_1": 0.728448, + "mrr_at_3": 0.802443, + "mrr_at_5": 0.811494, + "mrr_at_10": 0.81719, + "mrr_at_20": 0.818045, + "mrr_at_100": 0.818392, + "mrr_at_1000": 0.818486, + "nauc_ndcg_at_1_max": 0.386518, + "nauc_ndcg_at_1_std": 0.197838, + "nauc_ndcg_at_1_diff1": 0.386604, + "nauc_ndcg_at_3_max": 0.263702, + "nauc_ndcg_at_3_std": 0.23663, + "nauc_ndcg_at_3_diff1": 0.112831, + "nauc_ndcg_at_5_max": 0.254988, + "nauc_ndcg_at_5_std": 0.187844, + "nauc_ndcg_at_5_diff1": 0.071805, + "nauc_ndcg_at_10_max": 0.224638, + "nauc_ndcg_at_10_std": 0.194975, + "nauc_ndcg_at_10_diff1": 0.034801, + "nauc_ndcg_at_20_max": 0.215344, + "nauc_ndcg_at_20_std": 0.173884, + "nauc_ndcg_at_20_diff1": 0.043843, + "nauc_ndcg_at_100_max": 0.281461, + "nauc_ndcg_at_100_std": 0.132347, + "nauc_ndcg_at_100_diff1": 0.192721, + "nauc_ndcg_at_1000_max": 0.296281, + "nauc_ndcg_at_1000_std": 0.246238, + "nauc_ndcg_at_1000_diff1": 0.23313, + "nauc_map_at_1_max": 0.194813, + "nauc_map_at_1_std": -0.153977, + "nauc_map_at_1_diff1": 0.524228, + "nauc_map_at_3_max": 0.206424, + "nauc_map_at_3_std": -0.108054, + "nauc_map_at_3_diff1": 0.37963, + "nauc_map_at_5_max": 0.221944, + "nauc_map_at_5_std": -0.124836, + "nauc_map_at_5_diff1": 0.365137, + "nauc_map_at_10_max": 0.242095, + "nauc_map_at_10_std": -0.104948, + "nauc_map_at_10_diff1": 0.355487, + "nauc_map_at_20_max": 0.25949, + "nauc_map_at_20_std": -0.082268, + "nauc_map_at_20_diff1": 0.345232, + "nauc_map_at_100_max": 0.282829, + "nauc_map_at_100_std": 0.014433, + "nauc_map_at_100_diff1": 0.31476, + "nauc_map_at_1000_max": 0.280419, + "nauc_map_at_1000_std": 0.111395, + "nauc_map_at_1000_diff1": 0.257288, + "nauc_recall_at_1_max": 0.194813, + "nauc_recall_at_1_std": -0.153977, + "nauc_recall_at_1_diff1": 0.524228, + "nauc_recall_at_3_max": 0.181521, + "nauc_recall_at_3_std": -0.095824, + "nauc_recall_at_3_diff1": 0.352897, + "nauc_recall_at_5_max": 0.204961, + "nauc_recall_at_5_std": -0.127046, + "nauc_recall_at_5_diff1": 0.339212, + "nauc_recall_at_10_max": 0.233256, + "nauc_recall_at_10_std": -0.085916, + "nauc_recall_at_10_diff1": 0.331987, + "nauc_recall_at_20_max": 0.255122, + "nauc_recall_at_20_std": -0.065772, + "nauc_recall_at_20_diff1": 0.330324, + "nauc_recall_at_100_max": 0.281833, + "nauc_recall_at_100_std": 0.066131, + "nauc_recall_at_100_diff1": 0.287772, + "nauc_recall_at_1000_max": 0.271742, + "nauc_recall_at_1000_std": 0.241907, + "nauc_recall_at_1000_diff1": 0.227787, + "nauc_precision_at_1_max": 0.386518, + "nauc_precision_at_1_std": 0.197838, + "nauc_precision_at_1_diff1": 0.386604, + "nauc_precision_at_3_max": 0.208954, + "nauc_precision_at_3_std": 0.272296, + "nauc_precision_at_3_diff1": -0.038237, + "nauc_precision_at_5_max": 0.191793, + "nauc_precision_at_5_std": 0.207957, + "nauc_precision_at_5_diff1": -0.091209, + "nauc_precision_at_10_max": 0.10665, + "nauc_precision_at_10_std": 0.222863, + "nauc_precision_at_10_diff1": -0.189939, + "nauc_precision_at_20_max": 0.025146, + "nauc_precision_at_20_std": 0.229579, + "nauc_precision_at_20_diff1": -0.261522, + "nauc_precision_at_100_max": -0.033806, + "nauc_precision_at_100_std": 0.234683, + "nauc_precision_at_100_diff1": -0.259984, + "nauc_precision_at_1000_max": -0.094059, + "nauc_precision_at_1000_std": 0.09816, + "nauc_precision_at_1000_diff1": -0.269525, + "nauc_mrr_at_1_max": 0.386518, + "nauc_mrr_at_1_std": 0.197838, + "nauc_mrr_at_1_diff1": 0.386604, + "nauc_mrr_at_3_max": 0.290332, + "nauc_mrr_at_3_std": 0.200182, + "nauc_mrr_at_3_diff1": 0.31811, + "nauc_mrr_at_5_max": 0.316516, + "nauc_mrr_at_5_std": 0.183798, + "nauc_mrr_at_5_diff1": 0.317542, + "nauc_mrr_at_10_max": 0.319986, + "nauc_mrr_at_10_std": 0.188609, + "nauc_mrr_at_10_diff1": 0.331005, + "nauc_mrr_at_20_max": 0.322116, + "nauc_mrr_at_20_std": 0.190696, + "nauc_mrr_at_20_diff1": 0.335036, + "nauc_mrr_at_100_max": 0.322929, + "nauc_mrr_at_100_std": 0.191013, + "nauc_mrr_at_100_diff1": 0.334407, + "nauc_mrr_at_1000_max": 0.322863, + "nauc_mrr_at_1000_std": 0.190938, + "nauc_mrr_at_1000_diff1": 0.334311, + "hit_rate_at_1": 0.72845, + "hit_rate_at_3": 0.88793, + "hit_rate_at_5": 0.92672, + "hit_rate_at_10": 0.96552, + "hit_rate_at_20": 0.97845, + "hit_rate_at_100": 0.98707, + "hit_rate_at_1000": 1.0, + "main_score": 0.56309, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 213.57701802253723, + "kg_co2_emissions": null, + "date": 1777843410.190723 +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/GlobalNewsRetrieval.json b/results/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/GlobalNewsRetrieval.json new file mode 100644 index 0000000000..40d8072876 --- /dev/null +++ b/results/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/GlobalNewsRetrieval.json @@ -0,0 +1,1562 @@ +{ + "dataset_revision": "234dcd32922d32c78e08bfe506a2b90a5401cf7c", + "task_name": "GlobalNewsRetrieval", + "mteb_version": "2.12.38", + "scores": { + "test": [ + { + "ndcg_at_1": 0.61278, + "ndcg_at_3": 0.69899, + "ndcg_at_5": 0.72794, + "ndcg_at_10": 0.74776, + "ndcg_at_20": 0.75797, + "ndcg_at_100": 0.76994, + "ndcg_at_1000": 0.77209, + "map_at_1": 0.58271, + "map_at_3": 0.66879, + "map_at_5": 0.68651, + "map_at_10": 0.69602, + "map_at_20": 0.69899, + "map_at_100": 0.70093, + "map_at_1000": 0.70106, + "recall_at_1": 0.58271, + "recall_at_3": 0.76128, + "recall_at_5": 0.82853, + "recall_at_10": 0.88596, + "recall_at_20": 0.92502, + "recall_at_100": 0.98517, + "recall_at_1000": 1.0, + "accuracy": 0.58271, + "precision_at_1": 0.61278, + "precision_at_3": 0.27652, + "precision_at_5": 0.18195, + "precision_at_10": 0.09875, + "precision_at_20": 0.05157, + "precision_at_100": 0.01112, + "precision_at_1000": 0.00113, + "mrr_at_1": 0.612782, + "mrr_at_3": 0.693609, + "mrr_at_5": 0.707769, + "mrr_at_10": 0.713928, + "mrr_at_20": 0.716703, + "mrr_at_100": 0.718002, + "mrr_at_1000": 0.718072, + "nauc_ndcg_at_1_max": 0.375562, + "nauc_ndcg_at_1_std": -0.186367, + "nauc_ndcg_at_1_diff1": 0.701645, + "nauc_ndcg_at_3_max": 0.404484, + "nauc_ndcg_at_3_std": -0.209877, + "nauc_ndcg_at_3_diff1": 0.684781, + "nauc_ndcg_at_5_max": 0.409292, + "nauc_ndcg_at_5_std": -0.228861, + "nauc_ndcg_at_5_diff1": 0.6749, + "nauc_ndcg_at_10_max": 0.414023, + "nauc_ndcg_at_10_std": -0.204821, + "nauc_ndcg_at_10_diff1": 0.668119, + "nauc_ndcg_at_20_max": 0.409647, + "nauc_ndcg_at_20_std": -0.196502, + "nauc_ndcg_at_20_diff1": 0.672241, + "nauc_ndcg_at_100_max": 0.411946, + "nauc_ndcg_at_100_std": -0.194633, + "nauc_ndcg_at_100_diff1": 0.678643, + "nauc_ndcg_at_1000_max": 0.407963, + "nauc_ndcg_at_1000_std": -0.19957, + "nauc_ndcg_at_1000_diff1": 0.679789, + "nauc_map_at_1_max": 0.343123, + "nauc_map_at_1_std": -0.18701, + "nauc_map_at_1_diff1": 0.710892, + "nauc_map_at_3_max": 0.387488, + "nauc_map_at_3_std": -0.208742, + "nauc_map_at_3_diff1": 0.69602, + "nauc_map_at_5_max": 0.392681, + "nauc_map_at_5_std": -0.219404, + "nauc_map_at_5_diff1": 0.691068, + "nauc_map_at_10_max": 0.397865, + "nauc_map_at_10_std": -0.208612, + "nauc_map_at_10_diff1": 0.688687, + "nauc_map_at_20_max": 0.396497, + "nauc_map_at_20_std": -0.206741, + "nauc_map_at_20_diff1": 0.689823, + "nauc_map_at_100_max": 0.396506, + "nauc_map_at_100_std": -0.206189, + "nauc_map_at_100_diff1": 0.690164, + "nauc_map_at_1000_max": 0.396329, + "nauc_map_at_1000_std": -0.206388, + "nauc_map_at_1000_diff1": 0.690249, + "nauc_recall_at_1_max": 0.343123, + "nauc_recall_at_1_std": -0.18701, + "nauc_recall_at_1_diff1": 0.710892, + "nauc_recall_at_3_max": 0.419932, + "nauc_recall_at_3_std": -0.228622, + "nauc_recall_at_3_diff1": 0.655421, + "nauc_recall_at_5_max": 0.429836, + "nauc_recall_at_5_std": -0.293704, + "nauc_recall_at_5_diff1": 0.608509, + "nauc_recall_at_10_max": 0.462143, + "nauc_recall_at_10_std": -0.196918, + "nauc_recall_at_10_diff1": 0.561638, + "nauc_recall_at_20_max": 0.437083, + "nauc_recall_at_20_std": -0.105638, + "nauc_recall_at_20_diff1": 0.537565, + "nauc_recall_at_100_max": 0.762562, + "nauc_recall_at_100_std": 0.247457, + "nauc_recall_at_100_diff1": 0.587172, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.375562, + "nauc_precision_at_1_std": -0.186367, + "nauc_precision_at_1_diff1": 0.701645, + "nauc_precision_at_3_max": 0.427346, + "nauc_precision_at_3_std": -0.148041, + "nauc_precision_at_3_diff1": 0.434498, + "nauc_precision_at_5_max": 0.392885, + "nauc_precision_at_5_std": -0.148532, + "nauc_precision_at_5_diff1": 0.266093, + "nauc_precision_at_10_max": 0.298628, + "nauc_precision_at_10_std": -0.018057, + "nauc_precision_at_10_diff1": 0.059846, + "nauc_precision_at_20_max": 0.242568, + "nauc_precision_at_20_std": 0.041452, + "nauc_precision_at_20_diff1": -0.042218, + "nauc_precision_at_100_max": 0.137292, + "nauc_precision_at_100_std": 0.144693, + "nauc_precision_at_100_diff1": -0.222355, + "nauc_precision_at_1000_max": 0.074576, + "nauc_precision_at_1000_std": 0.124424, + "nauc_precision_at_1000_diff1": -0.267308, + "nauc_mrr_at_1_max": 0.375562, + "nauc_mrr_at_1_std": -0.186367, + "nauc_mrr_at_1_diff1": 0.701645, + "nauc_mrr_at_3_max": 0.414262, + "nauc_mrr_at_3_std": -0.192924, + "nauc_mrr_at_3_diff1": 0.675681, + "nauc_mrr_at_5_max": 0.412545, + "nauc_mrr_at_5_std": -0.200751, + "nauc_mrr_at_5_diff1": 0.671699, + "nauc_mrr_at_10_max": 0.410258, + "nauc_mrr_at_10_std": -0.194944, + "nauc_mrr_at_10_diff1": 0.670825, + "nauc_mrr_at_20_max": 0.409262, + "nauc_mrr_at_20_std": -0.192096, + "nauc_mrr_at_20_diff1": 0.67236, + "nauc_mrr_at_100_max": 0.409177, + "nauc_mrr_at_100_std": -0.192802, + "nauc_mrr_at_100_diff1": 0.673197, + "nauc_mrr_at_1000_max": 0.409055, + "nauc_mrr_at_1000_std": -0.192952, + "nauc_mrr_at_1000_diff1": 0.673212, + "hit_rate_at_1": 0.61278, + "hit_rate_at_3": 0.79198, + "hit_rate_at_5": 0.85338, + "hit_rate_at_10": 0.89975, + "hit_rate_at_20": 0.9386, + "hit_rate_at_100": 0.98872, + "hit_rate_at_1000": 1.0, + "main_score": 0.74776, + "hf_subset": "ar", + "languages": [ + "ara-Arab" + ] + }, + { + "ndcg_at_1": 0.75161, + "ndcg_at_3": 0.79692, + "ndcg_at_5": 0.81512, + "ndcg_at_10": 0.83113, + "ndcg_at_20": 0.8373, + "ndcg_at_100": 0.84586, + "ndcg_at_1000": 0.8485, + "map_at_1": 0.69606, + "map_at_3": 0.77569, + "map_at_5": 0.78731, + "map_at_10": 0.79496, + "map_at_20": 0.79711, + "map_at_100": 0.79861, + "map_at_1000": 0.79876, + "recall_at_1": 0.69606, + "recall_at_3": 0.83127, + "recall_at_5": 0.87285, + "recall_at_10": 0.91828, + "recall_at_20": 0.94014, + "recall_at_100": 0.98154, + "recall_at_1000": 1.0, + "accuracy": 0.69606, + "precision_at_1": 0.75161, + "precision_at_3": 0.3147, + "precision_at_5": 0.19978, + "precision_at_10": 0.10613, + "precision_at_20": 0.05484, + "precision_at_100": 0.01159, + "precision_at_1000": 0.00118, + "mrr_at_1": 0.751613, + "mrr_at_3": 0.802688, + "mrr_at_5": 0.810914, + "mrr_at_10": 0.81639, + "mrr_at_20": 0.817448, + "mrr_at_100": 0.818263, + "mrr_at_1000": 0.818345, + "nauc_ndcg_at_1_max": 0.513158, + "nauc_ndcg_at_1_std": -0.378559, + "nauc_ndcg_at_1_diff1": 0.766095, + "nauc_ndcg_at_3_max": 0.500464, + "nauc_ndcg_at_3_std": -0.416638, + "nauc_ndcg_at_3_diff1": 0.741717, + "nauc_ndcg_at_5_max": 0.507658, + "nauc_ndcg_at_5_std": -0.461707, + "nauc_ndcg_at_5_diff1": 0.745896, + "nauc_ndcg_at_10_max": 0.511083, + "nauc_ndcg_at_10_std": -0.473104, + "nauc_ndcg_at_10_diff1": 0.750668, + "nauc_ndcg_at_20_max": 0.511716, + "nauc_ndcg_at_20_std": -0.453317, + "nauc_ndcg_at_20_diff1": 0.750809, + "nauc_ndcg_at_100_max": 0.511435, + "nauc_ndcg_at_100_std": -0.434406, + "nauc_ndcg_at_100_diff1": 0.750334, + "nauc_ndcg_at_1000_max": 0.512762, + "nauc_ndcg_at_1000_std": -0.429752, + "nauc_ndcg_at_1000_diff1": 0.750332, + "nauc_map_at_1_max": 0.406706, + "nauc_map_at_1_std": -0.362253, + "nauc_map_at_1_diff1": 0.74563, + "nauc_map_at_3_max": 0.489247, + "nauc_map_at_3_std": -0.404131, + "nauc_map_at_3_diff1": 0.742981, + "nauc_map_at_5_max": 0.497152, + "nauc_map_at_5_std": -0.428898, + "nauc_map_at_5_diff1": 0.74695, + "nauc_map_at_10_max": 0.499282, + "nauc_map_at_10_std": -0.432501, + "nauc_map_at_10_diff1": 0.749295, + "nauc_map_at_20_max": 0.499693, + "nauc_map_at_20_std": -0.428118, + "nauc_map_at_20_diff1": 0.749219, + "nauc_map_at_100_max": 0.499743, + "nauc_map_at_100_std": -0.424592, + "nauc_map_at_100_diff1": 0.748917, + "nauc_map_at_1000_max": 0.499888, + "nauc_map_at_1000_std": -0.424318, + "nauc_map_at_1000_diff1": 0.748923, + "nauc_recall_at_1_max": 0.406706, + "nauc_recall_at_1_std": -0.362253, + "nauc_recall_at_1_diff1": 0.74563, + "nauc_recall_at_3_max": 0.48767, + "nauc_recall_at_3_std": -0.450813, + "nauc_recall_at_3_diff1": 0.711727, + "nauc_recall_at_5_max": 0.497573, + "nauc_recall_at_5_std": -0.609446, + "nauc_recall_at_5_diff1": 0.716402, + "nauc_recall_at_10_max": 0.494664, + "nauc_recall_at_10_std": -0.75312, + "nauc_recall_at_10_diff1": 0.719892, + "nauc_recall_at_20_max": 0.496644, + "nauc_recall_at_20_std": -0.680519, + "nauc_recall_at_20_diff1": 0.723732, + "nauc_recall_at_100_max": 0.470421, + "nauc_recall_at_100_std": -0.618563, + "nauc_recall_at_100_diff1": 0.741676, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.513158, + "nauc_precision_at_1_std": -0.378559, + "nauc_precision_at_1_diff1": 0.766095, + "nauc_precision_at_3_max": 0.361862, + "nauc_precision_at_3_std": -0.206392, + "nauc_precision_at_3_diff1": 0.235784, + "nauc_precision_at_5_max": 0.306077, + "nauc_precision_at_5_std": -0.209504, + "nauc_precision_at_5_diff1": 0.12219, + "nauc_precision_at_10_max": 0.216917, + "nauc_precision_at_10_std": -0.135808, + "nauc_precision_at_10_diff1": -0.030238, + "nauc_precision_at_20_max": 0.156974, + "nauc_precision_at_20_std": -0.014978, + "nauc_precision_at_20_diff1": -0.117606, + "nauc_precision_at_100_max": 0.051722, + "nauc_precision_at_100_std": 0.148774, + "nauc_precision_at_100_diff1": -0.263137, + "nauc_precision_at_1000_max": 0.020507, + "nauc_precision_at_1000_std": 0.205676, + "nauc_precision_at_1000_diff1": -0.323556, + "nauc_mrr_at_1_max": 0.513158, + "nauc_mrr_at_1_std": -0.378559, + "nauc_mrr_at_1_diff1": 0.766095, + "nauc_mrr_at_3_max": 0.525163, + "nauc_mrr_at_3_std": -0.405118, + "nauc_mrr_at_3_diff1": 0.756987, + "nauc_mrr_at_5_max": 0.524619, + "nauc_mrr_at_5_std": -0.421202, + "nauc_mrr_at_5_diff1": 0.755597, + "nauc_mrr_at_10_max": 0.526356, + "nauc_mrr_at_10_std": -0.421101, + "nauc_mrr_at_10_diff1": 0.757549, + "nauc_mrr_at_20_max": 0.526162, + "nauc_mrr_at_20_std": -0.416479, + "nauc_mrr_at_20_diff1": 0.757862, + "nauc_mrr_at_100_max": 0.526023, + "nauc_mrr_at_100_std": -0.41533, + "nauc_mrr_at_100_diff1": 0.757627, + "nauc_mrr_at_1000_max": 0.526011, + "nauc_mrr_at_1000_std": -0.415107, + "nauc_mrr_at_1000_diff1": 0.757622, + "hit_rate_at_1": 0.75161, + "hit_rate_at_3": 0.86237, + "hit_rate_at_5": 0.89892, + "hit_rate_at_10": 0.93978, + "hit_rate_at_20": 0.95591, + "hit_rate_at_100": 0.98602, + "hit_rate_at_1000": 1.0, + "main_score": 0.83113, + "hf_subset": "zh", + "languages": [ + "zho-Hans" + ] + }, + { + "ndcg_at_1": 0.64234, + "ndcg_at_3": 0.71225, + "ndcg_at_5": 0.73545, + "ndcg_at_10": 0.75605, + "ndcg_at_20": 0.76801, + "ndcg_at_100": 0.77963, + "ndcg_at_1000": 0.78231, + "map_at_1": 0.61913, + "map_at_3": 0.68868, + "map_at_5": 0.70212, + "map_at_10": 0.71121, + "map_at_20": 0.71504, + "map_at_100": 0.717, + "map_at_1000": 0.71713, + "recall_at_1": 0.61913, + "recall_at_3": 0.76387, + "recall_at_5": 0.81817, + "recall_at_10": 0.87984, + "recall_at_20": 0.92343, + "recall_at_100": 0.98073, + "recall_at_1000": 1.0, + "accuracy": 0.61913, + "precision_at_1": 0.64234, + "precision_at_3": 0.26927, + "precision_at_5": 0.1739, + "precision_at_10": 0.09414, + "precision_at_20": 0.04979, + "precision_at_100": 0.01071, + "precision_at_1000": 0.00109, + "mrr_at_1": 0.642343, + "mrr_at_3": 0.705721, + "mrr_at_5": 0.718671, + "mrr_at_10": 0.726246, + "mrr_at_20": 0.729168, + "mrr_at_100": 0.730552, + "mrr_at_1000": 0.73066, + "nauc_ndcg_at_1_max": 0.444309, + "nauc_ndcg_at_1_std": -0.264165, + "nauc_ndcg_at_1_diff1": 0.763114, + "nauc_ndcg_at_3_max": 0.461656, + "nauc_ndcg_at_3_std": -0.313958, + "nauc_ndcg_at_3_diff1": 0.716868, + "nauc_ndcg_at_5_max": 0.475807, + "nauc_ndcg_at_5_std": -0.32908, + "nauc_ndcg_at_5_diff1": 0.716142, + "nauc_ndcg_at_10_max": 0.479092, + "nauc_ndcg_at_10_std": -0.32335, + "nauc_ndcg_at_10_diff1": 0.713247, + "nauc_ndcg_at_20_max": 0.481675, + "nauc_ndcg_at_20_std": -0.313617, + "nauc_ndcg_at_20_diff1": 0.720608, + "nauc_ndcg_at_100_max": 0.474643, + "nauc_ndcg_at_100_std": -0.302845, + "nauc_ndcg_at_100_diff1": 0.726343, + "nauc_ndcg_at_1000_max": 0.472473, + "nauc_ndcg_at_1000_std": -0.304088, + "nauc_ndcg_at_1000_diff1": 0.726491, + "nauc_map_at_1_max": 0.417127, + "nauc_map_at_1_std": -0.272653, + "nauc_map_at_1_diff1": 0.753083, + "nauc_map_at_3_max": 0.450401, + "nauc_map_at_3_std": -0.307884, + "nauc_map_at_3_diff1": 0.726076, + "nauc_map_at_5_max": 0.460063, + "nauc_map_at_5_std": -0.31528, + "nauc_map_at_5_diff1": 0.726309, + "nauc_map_at_10_max": 0.462327, + "nauc_map_at_10_std": -0.312349, + "nauc_map_at_10_diff1": 0.725558, + "nauc_map_at_20_max": 0.463154, + "nauc_map_at_20_std": -0.308741, + "nauc_map_at_20_diff1": 0.727586, + "nauc_map_at_100_max": 0.462285, + "nauc_map_at_100_std": -0.307405, + "nauc_map_at_100_diff1": 0.72853, + "nauc_map_at_1000_max": 0.462205, + "nauc_map_at_1000_std": -0.307405, + "nauc_map_at_1000_diff1": 0.728516, + "nauc_recall_at_1_max": 0.417127, + "nauc_recall_at_1_std": -0.272653, + "nauc_recall_at_1_diff1": 0.753083, + "nauc_recall_at_3_max": 0.477158, + "nauc_recall_at_3_std": -0.356185, + "nauc_recall_at_3_diff1": 0.673322, + "nauc_recall_at_5_max": 0.511164, + "nauc_recall_at_5_std": -0.408612, + "nauc_recall_at_5_diff1": 0.651251, + "nauc_recall_at_10_max": 0.54028, + "nauc_recall_at_10_std": -0.4205, + "nauc_recall_at_10_diff1": 0.609245, + "nauc_recall_at_20_max": 0.591288, + "nauc_recall_at_20_std": -0.396628, + "nauc_recall_at_20_diff1": 0.623615, + "nauc_recall_at_100_max": 0.620603, + "nauc_recall_at_100_std": -0.194172, + "nauc_recall_at_100_diff1": 0.695541, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.444309, + "nauc_precision_at_1_std": -0.264165, + "nauc_precision_at_1_diff1": 0.763114, + "nauc_precision_at_3_max": 0.462919, + "nauc_precision_at_3_std": -0.263734, + "nauc_precision_at_3_diff1": 0.526736, + "nauc_precision_at_5_max": 0.477491, + "nauc_precision_at_5_std": -0.263847, + "nauc_precision_at_5_diff1": 0.437672, + "nauc_precision_at_10_max": 0.404105, + "nauc_precision_at_10_std": -0.171303, + "nauc_precision_at_10_diff1": 0.249716, + "nauc_precision_at_20_max": 0.302557, + "nauc_precision_at_20_std": -0.00088, + "nauc_precision_at_20_diff1": 0.080077, + "nauc_precision_at_100_max": 0.065924, + "nauc_precision_at_100_std": 0.212497, + "nauc_precision_at_100_diff1": -0.168379, + "nauc_precision_at_1000_max": -0.015513, + "nauc_precision_at_1000_std": 0.253782, + "nauc_precision_at_1000_diff1": -0.274816, + "nauc_mrr_at_1_max": 0.444309, + "nauc_mrr_at_1_std": -0.264165, + "nauc_mrr_at_1_diff1": 0.763114, + "nauc_mrr_at_3_max": 0.466179, + "nauc_mrr_at_3_std": -0.288409, + "nauc_mrr_at_3_diff1": 0.733057, + "nauc_mrr_at_5_max": 0.472326, + "nauc_mrr_at_5_std": -0.295391, + "nauc_mrr_at_5_diff1": 0.733044, + "nauc_mrr_at_10_max": 0.471704, + "nauc_mrr_at_10_std": -0.292014, + "nauc_mrr_at_10_diff1": 0.732823, + "nauc_mrr_at_20_max": 0.471677, + "nauc_mrr_at_20_std": -0.291431, + "nauc_mrr_at_20_diff1": 0.735122, + "nauc_mrr_at_100_max": 0.470759, + "nauc_mrr_at_100_std": -0.29004, + "nauc_mrr_at_100_diff1": 0.735573, + "nauc_mrr_at_1000_max": 0.470698, + "nauc_mrr_at_1000_std": -0.290094, + "nauc_mrr_at_1000_diff1": 0.735594, + "hit_rate_at_1": 0.64234, + "hit_rate_at_3": 0.78417, + "hit_rate_at_5": 0.83967, + "hit_rate_at_10": 0.89723, + "hit_rate_at_20": 0.93525, + "hit_rate_at_100": 0.98253, + "hit_rate_at_1000": 1.0, + "main_score": 0.75605, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.59653, + "ndcg_at_3": 0.68889, + "ndcg_at_5": 0.71085, + "ndcg_at_10": 0.73323, + "ndcg_at_20": 0.7452, + "ndcg_at_100": 0.7557, + "ndcg_at_1000": 0.75969, + "map_at_1": 0.57967, + "map_at_3": 0.66133, + "map_at_5": 0.67447, + "map_at_10": 0.68428, + "map_at_20": 0.68798, + "map_at_100": 0.68964, + "map_at_1000": 0.68984, + "recall_at_1": 0.57967, + "recall_at_3": 0.75434, + "recall_at_5": 0.80558, + "recall_at_10": 0.87249, + "recall_at_20": 0.9176, + "recall_at_100": 0.9714, + "recall_at_1000": 1.0, + "accuracy": 0.57967, + "precision_at_1": 0.59653, + "precision_at_3": 0.26319, + "precision_at_5": 0.16977, + "precision_at_10": 0.09254, + "precision_at_20": 0.04903, + "precision_at_100": 0.01042, + "precision_at_1000": 0.00107, + "mrr_at_1": 0.596527, + "mrr_at_3": 0.67586, + "mrr_at_5": 0.68684, + "mrr_at_10": 0.695391, + "mrr_at_20": 0.698113, + "mrr_at_100": 0.699478, + "mrr_at_1000": 0.699659, + "nauc_ndcg_at_1_max": 0.448288, + "nauc_ndcg_at_1_std": -0.212537, + "nauc_ndcg_at_1_diff1": 0.702626, + "nauc_ndcg_at_3_max": 0.453231, + "nauc_ndcg_at_3_std": -0.248775, + "nauc_ndcg_at_3_diff1": 0.669755, + "nauc_ndcg_at_5_max": 0.454878, + "nauc_ndcg_at_5_std": -0.260908, + "nauc_ndcg_at_5_diff1": 0.663544, + "nauc_ndcg_at_10_max": 0.460219, + "nauc_ndcg_at_10_std": -0.255246, + "nauc_ndcg_at_10_diff1": 0.666876, + "nauc_ndcg_at_20_max": 0.465581, + "nauc_ndcg_at_20_std": -0.250537, + "nauc_ndcg_at_20_diff1": 0.670021, + "nauc_ndcg_at_100_max": 0.461618, + "nauc_ndcg_at_100_std": -0.244261, + "nauc_ndcg_at_100_diff1": 0.673302, + "nauc_ndcg_at_1000_max": 0.458384, + "nauc_ndcg_at_1000_std": -0.243726, + "nauc_ndcg_at_1000_diff1": 0.673139, + "nauc_map_at_1_max": 0.433496, + "nauc_map_at_1_std": -0.207091, + "nauc_map_at_1_diff1": 0.701899, + "nauc_map_at_3_max": 0.447716, + "nauc_map_at_3_std": -0.239358, + "nauc_map_at_3_diff1": 0.678883, + "nauc_map_at_5_max": 0.450168, + "nauc_map_at_5_std": -0.244269, + "nauc_map_at_5_diff1": 0.675539, + "nauc_map_at_10_max": 0.451457, + "nauc_map_at_10_std": -0.24245, + "nauc_map_at_10_diff1": 0.677302, + "nauc_map_at_20_max": 0.452969, + "nauc_map_at_20_std": -0.240746, + "nauc_map_at_20_diff1": 0.677844, + "nauc_map_at_100_max": 0.452436, + "nauc_map_at_100_std": -0.240067, + "nauc_map_at_100_diff1": 0.678102, + "nauc_map_at_1000_max": 0.452288, + "nauc_map_at_1000_std": -0.24004, + "nauc_map_at_1000_diff1": 0.678069, + "nauc_recall_at_1_max": 0.433496, + "nauc_recall_at_1_std": -0.207091, + "nauc_recall_at_1_diff1": 0.701899, + "nauc_recall_at_3_max": 0.454826, + "nauc_recall_at_3_std": -0.280245, + "nauc_recall_at_3_diff1": 0.628629, + "nauc_recall_at_5_max": 0.456608, + "nauc_recall_at_5_std": -0.328132, + "nauc_recall_at_5_diff1": 0.601846, + "nauc_recall_at_10_max": 0.492082, + "nauc_recall_at_10_std": -0.324279, + "nauc_recall_at_10_diff1": 0.595789, + "nauc_recall_at_20_max": 0.559502, + "nauc_recall_at_20_std": -0.328044, + "nauc_recall_at_20_diff1": 0.603734, + "nauc_recall_at_100_max": 0.614661, + "nauc_recall_at_100_std": -0.264438, + "nauc_recall_at_100_diff1": 0.663472, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.448288, + "nauc_precision_at_1_std": -0.212537, + "nauc_precision_at_1_diff1": 0.702626, + "nauc_precision_at_3_max": 0.42217, + "nauc_precision_at_3_std": -0.265717, + "nauc_precision_at_3_diff1": 0.53553, + "nauc_precision_at_5_max": 0.382793, + "nauc_precision_at_5_std": -0.260256, + "nauc_precision_at_5_diff1": 0.421835, + "nauc_precision_at_10_max": 0.317139, + "nauc_precision_at_10_std": -0.198547, + "nauc_precision_at_10_diff1": 0.284782, + "nauc_precision_at_20_max": 0.260064, + "nauc_precision_at_20_std": -0.094142, + "nauc_precision_at_20_diff1": 0.127789, + "nauc_precision_at_100_max": 0.093311, + "nauc_precision_at_100_std": 0.032317, + "nauc_precision_at_100_diff1": -0.089778, + "nauc_precision_at_1000_max": -0.040383, + "nauc_precision_at_1000_std": 0.105636, + "nauc_precision_at_1000_diff1": -0.2672, + "nauc_mrr_at_1_max": 0.448288, + "nauc_mrr_at_1_std": -0.212537, + "nauc_mrr_at_1_diff1": 0.702626, + "nauc_mrr_at_3_max": 0.458341, + "nauc_mrr_at_3_std": -0.238297, + "nauc_mrr_at_3_diff1": 0.678364, + "nauc_mrr_at_5_max": 0.456375, + "nauc_mrr_at_5_std": -0.245503, + "nauc_mrr_at_5_diff1": 0.676219, + "nauc_mrr_at_10_max": 0.458575, + "nauc_mrr_at_10_std": -0.242494, + "nauc_mrr_at_10_diff1": 0.677031, + "nauc_mrr_at_20_max": 0.458768, + "nauc_mrr_at_20_std": -0.241906, + "nauc_mrr_at_20_diff1": 0.67805, + "nauc_mrr_at_100_max": 0.458277, + "nauc_mrr_at_100_std": -0.240941, + "nauc_mrr_at_100_diff1": 0.678337, + "nauc_mrr_at_1000_max": 0.458144, + "nauc_mrr_at_1000_std": -0.240929, + "nauc_mrr_at_1000_diff1": 0.678315, + "hit_rate_at_1": 0.59653, + "hit_rate_at_3": 0.77324, + "hit_rate_at_5": 0.82125, + "hit_rate_at_10": 0.88458, + "hit_rate_at_20": 0.92441, + "hit_rate_at_100": 0.97344, + "hit_rate_at_1000": 1.0, + "main_score": 0.73323, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.69386, + "ndcg_at_3": 0.77096, + "ndcg_at_5": 0.78737, + "ndcg_at_10": 0.80312, + "ndcg_at_20": 0.81369, + "ndcg_at_100": 0.82145, + "ndcg_at_1000": 0.82293, + "map_at_1": 0.67143, + "map_at_3": 0.74868, + "map_at_5": 0.75827, + "map_at_10": 0.76551, + "map_at_20": 0.76877, + "map_at_100": 0.76991, + "map_at_1000": 0.76998, + "recall_at_1": 0.67143, + "recall_at_3": 0.82398, + "recall_at_5": 0.86246, + "recall_at_10": 0.90881, + "recall_at_20": 0.94827, + "recall_at_100": 0.98905, + "recall_at_1000": 1.0, + "accuracy": 0.67143, + "precision_at_1": 0.69386, + "precision_at_3": 0.2899, + "precision_at_5": 0.18263, + "precision_at_10": 0.09672, + "precision_at_20": 0.05069, + "precision_at_100": 0.0106, + "precision_at_1000": 0.00107, + "mrr_at_1": 0.693856, + "mrr_at_3": 0.759887, + "mrr_at_5": 0.76852, + "mrr_at_10": 0.774326, + "mrr_at_20": 0.77697, + "mrr_at_100": 0.777931, + "mrr_at_1000": 0.777986, + "nauc_ndcg_at_1_max": 0.478653, + "nauc_ndcg_at_1_std": -0.245076, + "nauc_ndcg_at_1_diff1": 0.785322, + "nauc_ndcg_at_3_max": 0.497558, + "nauc_ndcg_at_3_std": -0.299383, + "nauc_ndcg_at_3_diff1": 0.753548, + "nauc_ndcg_at_5_max": 0.489087, + "nauc_ndcg_at_5_std": -0.325705, + "nauc_ndcg_at_5_diff1": 0.747592, + "nauc_ndcg_at_10_max": 0.498794, + "nauc_ndcg_at_10_std": -0.325129, + "nauc_ndcg_at_10_diff1": 0.752171, + "nauc_ndcg_at_20_max": 0.498459, + "nauc_ndcg_at_20_std": -0.323142, + "nauc_ndcg_at_20_diff1": 0.755446, + "nauc_ndcg_at_100_max": 0.492441, + "nauc_ndcg_at_100_std": -0.308709, + "nauc_ndcg_at_100_diff1": 0.757978, + "nauc_ndcg_at_1000_max": 0.491899, + "nauc_ndcg_at_1000_std": -0.30659, + "nauc_ndcg_at_1000_diff1": 0.758363, + "nauc_map_at_1_max": 0.433767, + "nauc_map_at_1_std": -0.235234, + "nauc_map_at_1_diff1": 0.773291, + "nauc_map_at_3_max": 0.484888, + "nauc_map_at_3_std": -0.2859, + "nauc_map_at_3_diff1": 0.757959, + "nauc_map_at_5_max": 0.479866, + "nauc_map_at_5_std": -0.30053, + "nauc_map_at_5_diff1": 0.755263, + "nauc_map_at_10_max": 0.484108, + "nauc_map_at_10_std": -0.299162, + "nauc_map_at_10_diff1": 0.757843, + "nauc_map_at_20_max": 0.483715, + "nauc_map_at_20_std": -0.29879, + "nauc_map_at_20_diff1": 0.758907, + "nauc_map_at_100_max": 0.482956, + "nauc_map_at_100_std": -0.296886, + "nauc_map_at_100_diff1": 0.759155, + "nauc_map_at_1000_max": 0.482934, + "nauc_map_at_1000_std": -0.296764, + "nauc_map_at_1000_diff1": 0.759167, + "nauc_recall_at_1_max": 0.433767, + "nauc_recall_at_1_std": -0.235234, + "nauc_recall_at_1_diff1": 0.773291, + "nauc_recall_at_3_max": 0.519816, + "nauc_recall_at_3_std": -0.340816, + "nauc_recall_at_3_diff1": 0.718281, + "nauc_recall_at_5_max": 0.501092, + "nauc_recall_at_5_std": -0.432965, + "nauc_recall_at_5_diff1": 0.69152, + "nauc_recall_at_10_max": 0.557458, + "nauc_recall_at_10_std": -0.480645, + "nauc_recall_at_10_diff1": 0.690717, + "nauc_recall_at_20_max": 0.609526, + "nauc_recall_at_20_std": -0.573307, + "nauc_recall_at_20_diff1": 0.694899, + "nauc_recall_at_100_max": 0.571587, + "nauc_recall_at_100_std": -0.481342, + "nauc_recall_at_100_diff1": 0.718787, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.478653, + "nauc_precision_at_1_std": -0.245076, + "nauc_precision_at_1_diff1": 0.785322, + "nauc_precision_at_3_max": 0.459476, + "nauc_precision_at_3_std": -0.250601, + "nauc_precision_at_3_diff1": 0.4815, + "nauc_precision_at_5_max": 0.393161, + "nauc_precision_at_5_std": -0.296849, + "nauc_precision_at_5_diff1": 0.372598, + "nauc_precision_at_10_max": 0.340667, + "nauc_precision_at_10_std": -0.232675, + "nauc_precision_at_10_diff1": 0.22753, + "nauc_precision_at_20_max": 0.228761, + "nauc_precision_at_20_std": -0.175437, + "nauc_precision_at_20_diff1": 0.049464, + "nauc_precision_at_100_max": 0.065684, + "nauc_precision_at_100_std": -0.002615, + "nauc_precision_at_100_diff1": -0.17741, + "nauc_precision_at_1000_max": 0.019244, + "nauc_precision_at_1000_std": 0.047523, + "nauc_precision_at_1000_diff1": -0.258985, + "nauc_mrr_at_1_max": 0.478653, + "nauc_mrr_at_1_std": -0.245076, + "nauc_mrr_at_1_diff1": 0.785322, + "nauc_mrr_at_3_max": 0.499846, + "nauc_mrr_at_3_std": -0.290385, + "nauc_mrr_at_3_diff1": 0.766262, + "nauc_mrr_at_5_max": 0.495069, + "nauc_mrr_at_5_std": -0.302476, + "nauc_mrr_at_5_diff1": 0.763751, + "nauc_mrr_at_10_max": 0.497589, + "nauc_mrr_at_10_std": -0.302917, + "nauc_mrr_at_10_diff1": 0.764968, + "nauc_mrr_at_20_max": 0.497671, + "nauc_mrr_at_20_std": -0.300867, + "nauc_mrr_at_20_diff1": 0.76575, + "nauc_mrr_at_100_max": 0.497012, + "nauc_mrr_at_100_std": -0.29893, + "nauc_mrr_at_100_diff1": 0.766096, + "nauc_mrr_at_1000_max": 0.497003, + "nauc_mrr_at_1000_std": -0.298799, + "nauc_mrr_at_1000_diff1": 0.766118, + "hit_rate_at_1": 0.69386, + "hit_rate_at_3": 0.83686, + "hit_rate_at_5": 0.87394, + "hit_rate_at_10": 0.91631, + "hit_rate_at_20": 0.95233, + "hit_rate_at_100": 0.98941, + "hit_rate_at_1000": 1.0, + "main_score": 0.80312, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ] + }, + { + "ndcg_at_1": 0.55971, + "ndcg_at_3": 0.63229, + "ndcg_at_5": 0.6512, + "ndcg_at_10": 0.66993, + "ndcg_at_20": 0.68226, + "ndcg_at_100": 0.69753, + "ndcg_at_1000": 0.70893, + "map_at_1": 0.54327, + "map_at_3": 0.60981, + "map_at_5": 0.62045, + "map_at_10": 0.62857, + "map_at_20": 0.63207, + "map_at_100": 0.63432, + "map_at_1000": 0.63477, + "recall_at_1": 0.54327, + "recall_at_3": 0.68605, + "recall_at_5": 0.73088, + "recall_at_10": 0.7866, + "recall_at_20": 0.83403, + "recall_at_100": 0.91329, + "recall_at_1000": 1.0, + "accuracy": 0.54327, + "precision_at_1": 0.55971, + "precision_at_3": 0.23884, + "precision_at_5": 0.15306, + "precision_at_10": 0.08297, + "precision_at_20": 0.04408, + "precision_at_100": 0.00974, + "precision_at_1000": 0.00108, + "mrr_at_1": 0.559709, + "mrr_at_3": 0.621841, + "mrr_at_5": 0.632433, + "mrr_at_10": 0.639916, + "mrr_at_20": 0.643351, + "mrr_at_100": 0.645294, + "mrr_at_1000": 0.645645, + "nauc_ndcg_at_1_max": 0.439466, + "nauc_ndcg_at_1_std": -0.265231, + "nauc_ndcg_at_1_diff1": 0.751159, + "nauc_ndcg_at_3_max": 0.428263, + "nauc_ndcg_at_3_std": -0.288085, + "nauc_ndcg_at_3_diff1": 0.681098, + "nauc_ndcg_at_5_max": 0.424962, + "nauc_ndcg_at_5_std": -0.284361, + "nauc_ndcg_at_5_diff1": 0.678863, + "nauc_ndcg_at_10_max": 0.431391, + "nauc_ndcg_at_10_std": -0.275166, + "nauc_ndcg_at_10_diff1": 0.680651, + "nauc_ndcg_at_20_max": 0.43295, + "nauc_ndcg_at_20_std": -0.275267, + "nauc_ndcg_at_20_diff1": 0.680497, + "nauc_ndcg_at_100_max": 0.432273, + "nauc_ndcg_at_100_std": -0.269614, + "nauc_ndcg_at_100_diff1": 0.68151, + "nauc_ndcg_at_1000_max": 0.433196, + "nauc_ndcg_at_1000_std": -0.273819, + "nauc_ndcg_at_1000_diff1": 0.690072, + "nauc_map_at_1_max": 0.416912, + "nauc_map_at_1_std": -0.255657, + "nauc_map_at_1_diff1": 0.73257, + "nauc_map_at_3_max": 0.423758, + "nauc_map_at_3_std": -0.282393, + "nauc_map_at_3_diff1": 0.692054, + "nauc_map_at_5_max": 0.421965, + "nauc_map_at_5_std": -0.280077, + "nauc_map_at_5_diff1": 0.69148, + "nauc_map_at_10_max": 0.425465, + "nauc_map_at_10_std": -0.276216, + "nauc_map_at_10_diff1": 0.693042, + "nauc_map_at_20_max": 0.425743, + "nauc_map_at_20_std": -0.276251, + "nauc_map_at_20_diff1": 0.693173, + "nauc_map_at_100_max": 0.425486, + "nauc_map_at_100_std": -0.275785, + "nauc_map_at_100_diff1": 0.69313, + "nauc_map_at_1000_max": 0.425751, + "nauc_map_at_1000_std": -0.275748, + "nauc_map_at_1000_diff1": 0.693545, + "nauc_recall_at_1_max": 0.416912, + "nauc_recall_at_1_std": -0.255657, + "nauc_recall_at_1_diff1": 0.73257, + "nauc_recall_at_3_max": 0.423048, + "nauc_recall_at_3_std": -0.302633, + "nauc_recall_at_3_diff1": 0.62285, + "nauc_recall_at_5_max": 0.408935, + "nauc_recall_at_5_std": -0.290078, + "nauc_recall_at_5_diff1": 0.601954, + "nauc_recall_at_10_max": 0.428814, + "nauc_recall_at_10_std": -0.255953, + "nauc_recall_at_10_diff1": 0.594005, + "nauc_recall_at_20_max": 0.435082, + "nauc_recall_at_20_std": -0.255249, + "nauc_recall_at_20_diff1": 0.566375, + "nauc_recall_at_100_max": 0.431555, + "nauc_recall_at_100_std": -0.153584, + "nauc_recall_at_100_diff1": 0.498947, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.439466, + "nauc_precision_at_1_std": -0.265231, + "nauc_precision_at_1_diff1": 0.751159, + "nauc_precision_at_3_max": 0.415841, + "nauc_precision_at_3_std": -0.293647, + "nauc_precision_at_3_diff1": 0.57263, + "nauc_precision_at_5_max": 0.401146, + "nauc_precision_at_5_std": -0.280581, + "nauc_precision_at_5_diff1": 0.541853, + "nauc_precision_at_10_max": 0.392446, + "nauc_precision_at_10_std": -0.221389, + "nauc_precision_at_10_diff1": 0.479881, + "nauc_precision_at_20_max": 0.379139, + "nauc_precision_at_20_std": -0.189912, + "nauc_precision_at_20_diff1": 0.417316, + "nauc_precision_at_100_max": 0.242493, + "nauc_precision_at_100_std": -0.024422, + "nauc_precision_at_100_diff1": 0.174429, + "nauc_precision_at_1000_max": 0.004983, + "nauc_precision_at_1000_std": 0.08949, + "nauc_precision_at_1000_diff1": -0.132823, + "nauc_mrr_at_1_max": 0.439466, + "nauc_mrr_at_1_std": -0.265231, + "nauc_mrr_at_1_diff1": 0.751159, + "nauc_mrr_at_3_max": 0.441634, + "nauc_mrr_at_3_std": -0.283946, + "nauc_mrr_at_3_diff1": 0.711365, + "nauc_mrr_at_5_max": 0.439871, + "nauc_mrr_at_5_std": -0.282767, + "nauc_mrr_at_5_diff1": 0.710896, + "nauc_mrr_at_10_max": 0.441917, + "nauc_mrr_at_10_std": -0.279564, + "nauc_mrr_at_10_diff1": 0.711575, + "nauc_mrr_at_20_max": 0.442481, + "nauc_mrr_at_20_std": -0.278959, + "nauc_mrr_at_20_diff1": 0.711744, + "nauc_mrr_at_100_max": 0.442287, + "nauc_mrr_at_100_std": -0.278458, + "nauc_mrr_at_100_diff1": 0.711844, + "nauc_mrr_at_1000_max": 0.442385, + "nauc_mrr_at_1000_std": -0.278564, + "nauc_mrr_at_1000_diff1": 0.712132, + "hit_rate_at_1": 0.55971, + "hit_rate_at_3": 0.70093, + "hit_rate_at_5": 0.74663, + "hit_rate_at_10": 0.8027, + "hit_rate_at_20": 0.85151, + "hit_rate_at_100": 0.92523, + "hit_rate_at_1000": 1.0, + "main_score": 0.66993, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.7163, + "ndcg_at_3": 0.79722, + "ndcg_at_5": 0.81481, + "ndcg_at_10": 0.83064, + "ndcg_at_20": 0.83767, + "ndcg_at_100": 0.84327, + "ndcg_at_1000": 0.84364, + "map_at_1": 0.69777, + "map_at_3": 0.77428, + "map_at_5": 0.78473, + "map_at_10": 0.7919, + "map_at_20": 0.79412, + "map_at_100": 0.79505, + "map_at_1000": 0.79506, + "recall_at_1": 0.69777, + "recall_at_3": 0.85446, + "recall_at_5": 0.89621, + "recall_at_10": 0.94274, + "recall_at_20": 0.96906, + "recall_at_100": 0.99698, + "recall_at_1000": 1.0, + "accuracy": 0.69777, + "precision_at_1": 0.7163, + "precision_at_3": 0.29745, + "precision_at_5": 0.18793, + "precision_at_10": 0.0993, + "precision_at_20": 0.05131, + "precision_at_100": 0.01057, + "precision_at_1000": 0.00106, + "mrr_at_1": 0.716298, + "mrr_at_3": 0.783032, + "mrr_at_5": 0.791985, + "mrr_at_10": 0.798306, + "mrr_at_20": 0.799864, + "mrr_at_100": 0.800687, + "mrr_at_1000": 0.800698, + "nauc_ndcg_at_1_max": 0.471944, + "nauc_ndcg_at_1_std": -0.085614, + "nauc_ndcg_at_1_diff1": 0.807556, + "nauc_ndcg_at_3_max": 0.476252, + "nauc_ndcg_at_3_std": -0.121456, + "nauc_ndcg_at_3_diff1": 0.767278, + "nauc_ndcg_at_5_max": 0.470614, + "nauc_ndcg_at_5_std": -0.120712, + "nauc_ndcg_at_5_diff1": 0.7604, + "nauc_ndcg_at_10_max": 0.481167, + "nauc_ndcg_at_10_std": -0.106124, + "nauc_ndcg_at_10_diff1": 0.761141, + "nauc_ndcg_at_20_max": 0.488903, + "nauc_ndcg_at_20_std": -0.095394, + "nauc_ndcg_at_20_diff1": 0.769311, + "nauc_ndcg_at_100_max": 0.485095, + "nauc_ndcg_at_100_std": -0.092993, + "nauc_ndcg_at_100_diff1": 0.774594, + "nauc_ndcg_at_1000_max": 0.483751, + "nauc_ndcg_at_1000_std": -0.095723, + "nauc_ndcg_at_1000_diff1": 0.774111, + "nauc_map_at_1_max": 0.418478, + "nauc_map_at_1_std": -0.110116, + "nauc_map_at_1_diff1": 0.803391, + "nauc_map_at_3_max": 0.463627, + "nauc_map_at_3_std": -0.121367, + "nauc_map_at_3_diff1": 0.775817, + "nauc_map_at_5_max": 0.463584, + "nauc_map_at_5_std": -0.119084, + "nauc_map_at_5_diff1": 0.773399, + "nauc_map_at_10_max": 0.468823, + "nauc_map_at_10_std": -0.111008, + "nauc_map_at_10_diff1": 0.773986, + "nauc_map_at_20_max": 0.471917, + "nauc_map_at_20_std": -0.107035, + "nauc_map_at_20_diff1": 0.776148, + "nauc_map_at_100_max": 0.471429, + "nauc_map_at_100_std": -0.106323, + "nauc_map_at_100_diff1": 0.776805, + "nauc_map_at_1000_max": 0.471398, + "nauc_map_at_1000_std": -0.106382, + "nauc_map_at_1000_diff1": 0.776794, + "nauc_recall_at_1_max": 0.418478, + "nauc_recall_at_1_std": -0.110116, + "nauc_recall_at_1_diff1": 0.803391, + "nauc_recall_at_3_max": 0.477584, + "nauc_recall_at_3_std": -0.1633, + "nauc_recall_at_3_diff1": 0.721741, + "nauc_recall_at_5_max": 0.451996, + "nauc_recall_at_5_std": -0.177466, + "nauc_recall_at_5_diff1": 0.678139, + "nauc_recall_at_10_max": 0.504487, + "nauc_recall_at_10_std": -0.134689, + "nauc_recall_at_10_diff1": 0.619514, + "nauc_recall_at_20_max": 0.624129, + "nauc_recall_at_20_std": -0.012958, + "nauc_recall_at_20_diff1": 0.656553, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": 0.956446, + "nauc_recall_at_100_diff1": 0.956446, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.471944, + "nauc_precision_at_1_std": -0.085614, + "nauc_precision_at_1_diff1": 0.807556, + "nauc_precision_at_3_max": 0.471473, + "nauc_precision_at_3_std": -0.038292, + "nauc_precision_at_3_diff1": 0.449506, + "nauc_precision_at_5_max": 0.415433, + "nauc_precision_at_5_std": -0.005179, + "nauc_precision_at_5_diff1": 0.296055, + "nauc_precision_at_10_max": 0.389676, + "nauc_precision_at_10_std": 0.123603, + "nauc_precision_at_10_diff1": 0.06343, + "nauc_precision_at_20_max": 0.355091, + "nauc_precision_at_20_std": 0.255324, + "nauc_precision_at_20_diff1": -0.087378, + "nauc_precision_at_100_max": 0.257567, + "nauc_precision_at_100_std": 0.34994, + "nauc_precision_at_100_diff1": -0.284614, + "nauc_precision_at_1000_max": 0.236254, + "nauc_precision_at_1000_std": 0.33253, + "nauc_precision_at_1000_diff1": -0.32024, + "nauc_mrr_at_1_max": 0.471944, + "nauc_mrr_at_1_std": -0.085614, + "nauc_mrr_at_1_diff1": 0.807556, + "nauc_mrr_at_3_max": 0.495618, + "nauc_mrr_at_3_std": -0.094299, + "nauc_mrr_at_3_diff1": 0.781811, + "nauc_mrr_at_5_max": 0.489274, + "nauc_mrr_at_5_std": -0.093997, + "nauc_mrr_at_5_diff1": 0.778104, + "nauc_mrr_at_10_max": 0.492171, + "nauc_mrr_at_10_std": -0.088779, + "nauc_mrr_at_10_diff1": 0.77875, + "nauc_mrr_at_20_max": 0.492433, + "nauc_mrr_at_20_std": -0.088471, + "nauc_mrr_at_20_diff1": 0.780585, + "nauc_mrr_at_100_max": 0.491801, + "nauc_mrr_at_100_std": -0.08808, + "nauc_mrr_at_100_diff1": 0.781298, + "nauc_mrr_at_1000_max": 0.491771, + "nauc_mrr_at_1000_std": -0.08814, + "nauc_mrr_at_1000_diff1": 0.781287, + "hit_rate_at_1": 0.7163, + "hit_rate_at_3": 0.86419, + "hit_rate_at_5": 0.90342, + "hit_rate_at_10": 0.94869, + "hit_rate_at_20": 0.97082, + "hit_rate_at_100": 0.99698, + "hit_rate_at_1000": 1.0, + "main_score": 0.83064, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ] + }, + { + "ndcg_at_1": 0.58273, + "ndcg_at_3": 0.6668, + "ndcg_at_5": 0.69299, + "ndcg_at_10": 0.71359, + "ndcg_at_20": 0.72481, + "ndcg_at_100": 0.73926, + "ndcg_at_1000": 0.74442, + "map_at_1": 0.55758, + "map_at_3": 0.64155, + "map_at_5": 0.65691, + "map_at_10": 0.66617, + "map_at_20": 0.66946, + "map_at_100": 0.67176, + "map_at_1000": 0.672, + "recall_at_1": 0.55758, + "recall_at_3": 0.7232, + "recall_at_5": 0.78529, + "recall_at_10": 0.84582, + "recall_at_20": 0.88866, + "recall_at_100": 0.96237, + "recall_at_1000": 1.0, + "accuracy": 0.55758, + "precision_at_1": 0.58273, + "precision_at_3": 0.26084, + "precision_at_5": 0.17066, + "precision_at_10": 0.09272, + "precision_at_20": 0.0488, + "precision_at_100": 0.01066, + "precision_at_1000": 0.00111, + "mrr_at_1": 0.582726, + "mrr_at_3": 0.657475, + "mrr_at_5": 0.670794, + "mrr_at_10": 0.678329, + "mrr_at_20": 0.681252, + "mrr_at_100": 0.682982, + "mrr_at_1000": 0.683185, + "nauc_ndcg_at_1_max": 0.357731, + "nauc_ndcg_at_1_std": -0.018293, + "nauc_ndcg_at_1_diff1": 0.711154, + "nauc_ndcg_at_3_max": 0.373034, + "nauc_ndcg_at_3_std": 3e-05, + "nauc_ndcg_at_3_diff1": 0.673882, + "nauc_ndcg_at_5_max": 0.380958, + "nauc_ndcg_at_5_std": -0.004806, + "nauc_ndcg_at_5_diff1": 0.674373, + "nauc_ndcg_at_10_max": 0.381171, + "nauc_ndcg_at_10_std": 0.005017, + "nauc_ndcg_at_10_diff1": 0.678657, + "nauc_ndcg_at_20_max": 0.37999, + "nauc_ndcg_at_20_std": -0.002436, + "nauc_ndcg_at_20_diff1": 0.678578, + "nauc_ndcg_at_100_max": 0.38621, + "nauc_ndcg_at_100_std": 0.008425, + "nauc_ndcg_at_100_diff1": 0.68008, + "nauc_ndcg_at_1000_max": 0.381845, + "nauc_ndcg_at_1000_std": 0.002308, + "nauc_ndcg_at_1000_diff1": 0.681508, + "nauc_map_at_1_max": 0.320966, + "nauc_map_at_1_std": -0.030771, + "nauc_map_at_1_diff1": 0.704752, + "nauc_map_at_3_max": 0.36576, + "nauc_map_at_3_std": -0.008374, + "nauc_map_at_3_diff1": 0.681761, + "nauc_map_at_5_max": 0.370676, + "nauc_map_at_5_std": -0.011096, + "nauc_map_at_5_diff1": 0.681827, + "nauc_map_at_10_max": 0.372034, + "nauc_map_at_10_std": -0.005952, + "nauc_map_at_10_diff1": 0.684395, + "nauc_map_at_20_max": 0.372627, + "nauc_map_at_20_std": -0.007583, + "nauc_map_at_20_diff1": 0.684579, + "nauc_map_at_100_max": 0.373745, + "nauc_map_at_100_std": -0.005781, + "nauc_map_at_100_diff1": 0.684722, + "nauc_map_at_1000_max": 0.373504, + "nauc_map_at_1000_std": -0.006105, + "nauc_map_at_1000_diff1": 0.68474, + "nauc_recall_at_1_max": 0.320966, + "nauc_recall_at_1_std": -0.030771, + "nauc_recall_at_1_diff1": 0.704752, + "nauc_recall_at_3_max": 0.37229, + "nauc_recall_at_3_std": 0.012489, + "nauc_recall_at_3_diff1": 0.635833, + "nauc_recall_at_5_max": 0.394425, + "nauc_recall_at_5_std": 2.9e-05, + "nauc_recall_at_5_diff1": 0.632698, + "nauc_recall_at_10_max": 0.395093, + "nauc_recall_at_10_std": 0.040586, + "nauc_recall_at_10_diff1": 0.634595, + "nauc_recall_at_20_max": 0.377343, + "nauc_recall_at_20_std": -0.015871, + "nauc_recall_at_20_diff1": 0.622307, + "nauc_recall_at_100_max": 0.536113, + "nauc_recall_at_100_std": 0.230542, + "nauc_recall_at_100_diff1": 0.596248, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.357731, + "nauc_precision_at_1_std": -0.018293, + "nauc_precision_at_1_diff1": 0.711154, + "nauc_precision_at_3_max": 0.374112, + "nauc_precision_at_3_std": 0.031535, + "nauc_precision_at_3_diff1": 0.477702, + "nauc_precision_at_5_max": 0.366401, + "nauc_precision_at_5_std": 0.02546, + "nauc_precision_at_5_diff1": 0.396999, + "nauc_precision_at_10_max": 0.318641, + "nauc_precision_at_10_std": 0.071257, + "nauc_precision_at_10_diff1": 0.288981, + "nauc_precision_at_20_max": 0.296612, + "nauc_precision_at_20_std": 0.061761, + "nauc_precision_at_20_diff1": 0.200247, + "nauc_precision_at_100_max": 0.212305, + "nauc_precision_at_100_std": 0.14706, + "nauc_precision_at_100_diff1": -0.04435, + "nauc_precision_at_1000_max": 0.108205, + "nauc_precision_at_1000_std": 0.09969, + "nauc_precision_at_1000_diff1": -0.186022, + "nauc_mrr_at_1_max": 0.357731, + "nauc_mrr_at_1_std": -0.018293, + "nauc_mrr_at_1_diff1": 0.711154, + "nauc_mrr_at_3_max": 0.380705, + "nauc_mrr_at_3_std": 0.005898, + "nauc_mrr_at_3_diff1": 0.68404, + "nauc_mrr_at_5_max": 0.384593, + "nauc_mrr_at_5_std": 0.003272, + "nauc_mrr_at_5_diff1": 0.685157, + "nauc_mrr_at_10_max": 0.38175, + "nauc_mrr_at_10_std": 0.005171, + "nauc_mrr_at_10_diff1": 0.686105, + "nauc_mrr_at_20_max": 0.381053, + "nauc_mrr_at_20_std": 0.003432, + "nauc_mrr_at_20_diff1": 0.68598, + "nauc_mrr_at_100_max": 0.381558, + "nauc_mrr_at_100_std": 0.004301, + "nauc_mrr_at_100_diff1": 0.686069, + "nauc_mrr_at_1000_max": 0.381386, + "nauc_mrr_at_1000_std": 0.004087, + "nauc_mrr_at_1000_diff1": 0.686111, + "hit_rate_at_1": 0.58273, + "hit_rate_at_3": 0.74506, + "hit_rate_at_5": 0.80437, + "hit_rate_at_10": 0.86056, + "hit_rate_at_20": 0.90219, + "hit_rate_at_100": 0.9667, + "hit_rate_at_1000": 1.0, + "main_score": 0.71359, + "hf_subset": "sw", + "languages": [ + "swa-Latn" + ] + }, + { + "ndcg_at_1": 0.69222, + "ndcg_at_3": 0.78022, + "ndcg_at_5": 0.79624, + "ndcg_at_10": 0.80976, + "ndcg_at_20": 0.81847, + "ndcg_at_100": 0.82477, + "ndcg_at_1000": 0.82624, + "map_at_1": 0.67407, + "map_at_3": 0.75565, + "map_at_5": 0.76493, + "map_at_10": 0.77107, + "map_at_20": 0.77355, + "map_at_100": 0.77447, + "map_at_1000": 0.77454, + "recall_at_1": 0.67407, + "recall_at_3": 0.84278, + "recall_at_5": 0.88167, + "recall_at_10": 0.92204, + "recall_at_20": 0.95593, + "recall_at_100": 0.98944, + "recall_at_1000": 1.0, + "accuracy": 0.67407, + "precision_at_1": 0.69222, + "precision_at_3": 0.29444, + "precision_at_5": 0.18511, + "precision_at_10": 0.09711, + "precision_at_20": 0.05028, + "precision_at_100": 0.01041, + "precision_at_1000": 0.00105, + "mrr_at_1": 0.692222, + "mrr_at_3": 0.763889, + "mrr_at_5": 0.771722, + "mrr_at_10": 0.776497, + "mrr_at_20": 0.778905, + "mrr_at_100": 0.77973, + "mrr_at_1000": 0.77979, + "nauc_ndcg_at_1_max": 0.420572, + "nauc_ndcg_at_1_std": -0.177366, + "nauc_ndcg_at_1_diff1": 0.777121, + "nauc_ndcg_at_3_max": 0.435204, + "nauc_ndcg_at_3_std": -0.269471, + "nauc_ndcg_at_3_diff1": 0.751024, + "nauc_ndcg_at_5_max": 0.428314, + "nauc_ndcg_at_5_std": -0.273233, + "nauc_ndcg_at_5_diff1": 0.74556, + "nauc_ndcg_at_10_max": 0.433577, + "nauc_ndcg_at_10_std": -0.260125, + "nauc_ndcg_at_10_diff1": 0.745426, + "nauc_ndcg_at_20_max": 0.427186, + "nauc_ndcg_at_20_std": -0.248318, + "nauc_ndcg_at_20_diff1": 0.750091, + "nauc_ndcg_at_100_max": 0.426059, + "nauc_ndcg_at_100_std": -0.242215, + "nauc_ndcg_at_100_diff1": 0.750211, + "nauc_ndcg_at_1000_max": 0.428663, + "nauc_ndcg_at_1000_std": -0.241863, + "nauc_ndcg_at_1000_diff1": 0.753091, + "nauc_map_at_1_max": 0.384789, + "nauc_map_at_1_std": -0.178272, + "nauc_map_at_1_diff1": 0.771572, + "nauc_map_at_3_max": 0.424453, + "nauc_map_at_3_std": -0.2473, + "nauc_map_at_3_diff1": 0.760249, + "nauc_map_at_5_max": 0.422912, + "nauc_map_at_5_std": -0.248442, + "nauc_map_at_5_diff1": 0.757134, + "nauc_map_at_10_max": 0.425516, + "nauc_map_at_10_std": -0.241482, + "nauc_map_at_10_diff1": 0.757655, + "nauc_map_at_20_max": 0.423941, + "nauc_map_at_20_std": -0.238442, + "nauc_map_at_20_diff1": 0.758876, + "nauc_map_at_100_max": 0.423786, + "nauc_map_at_100_std": -0.237777, + "nauc_map_at_100_diff1": 0.758823, + "nauc_map_at_1000_max": 0.423879, + "nauc_map_at_1000_std": -0.237778, + "nauc_map_at_1000_diff1": 0.758914, + "nauc_recall_at_1_max": 0.384789, + "nauc_recall_at_1_std": -0.178272, + "nauc_recall_at_1_diff1": 0.771572, + "nauc_recall_at_3_max": 0.446483, + "nauc_recall_at_3_std": -0.373769, + "nauc_recall_at_3_diff1": 0.711699, + "nauc_recall_at_5_max": 0.420775, + "nauc_recall_at_5_std": -0.418907, + "nauc_recall_at_5_diff1": 0.686328, + "nauc_recall_at_10_max": 0.449898, + "nauc_recall_at_10_std": -0.4163, + "nauc_recall_at_10_diff1": 0.658278, + "nauc_recall_at_20_max": 0.372276, + "nauc_recall_at_20_std": -0.370633, + "nauc_recall_at_20_diff1": 0.658312, + "nauc_recall_at_100_max": 0.144282, + "nauc_recall_at_100_std": -0.261929, + "nauc_recall_at_100_diff1": 0.424935, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.420572, + "nauc_precision_at_1_std": -0.177366, + "nauc_precision_at_1_diff1": 0.777121, + "nauc_precision_at_3_max": 0.373496, + "nauc_precision_at_3_std": -0.293862, + "nauc_precision_at_3_diff1": 0.467546, + "nauc_precision_at_5_max": 0.321701, + "nauc_precision_at_5_std": -0.280453, + "nauc_precision_at_5_diff1": 0.350189, + "nauc_precision_at_10_max": 0.266084, + "nauc_precision_at_10_std": -0.185704, + "nauc_precision_at_10_diff1": 0.191992, + "nauc_precision_at_20_max": 0.183272, + "nauc_precision_at_20_std": -0.101123, + "nauc_precision_at_20_diff1": 0.058281, + "nauc_precision_at_100_max": 0.08309, + "nauc_precision_at_100_std": 0.033122, + "nauc_precision_at_100_diff1": -0.188318, + "nauc_precision_at_1000_max": 0.061896, + "nauc_precision_at_1000_std": 0.080734, + "nauc_precision_at_1000_diff1": -0.261722, + "nauc_mrr_at_1_max": 0.420572, + "nauc_mrr_at_1_std": -0.177366, + "nauc_mrr_at_1_diff1": 0.777121, + "nauc_mrr_at_3_max": 0.445931, + "nauc_mrr_at_3_std": -0.23268, + "nauc_mrr_at_3_diff1": 0.756158, + "nauc_mrr_at_5_max": 0.439158, + "nauc_mrr_at_5_std": -0.233607, + "nauc_mrr_at_5_diff1": 0.754543, + "nauc_mrr_at_10_max": 0.439944, + "nauc_mrr_at_10_std": -0.229609, + "nauc_mrr_at_10_diff1": 0.754312, + "nauc_mrr_at_20_max": 0.438428, + "nauc_mrr_at_20_std": -0.226506, + "nauc_mrr_at_20_diff1": 0.755591, + "nauc_mrr_at_100_max": 0.438168, + "nauc_mrr_at_100_std": -0.226198, + "nauc_mrr_at_100_diff1": 0.755647, + "nauc_mrr_at_1000_max": 0.438299, + "nauc_mrr_at_1000_std": -0.226169, + "nauc_mrr_at_1000_diff1": 0.755752, + "hit_rate_at_1": 0.69222, + "hit_rate_at_3": 0.85333, + "hit_rate_at_5": 0.88889, + "hit_rate_at_10": 0.92444, + "hit_rate_at_20": 0.95778, + "hit_rate_at_100": 0.99, + "hit_rate_at_1000": 1.0, + "main_score": 0.80976, + "hf_subset": "hi", + "languages": [ + "hin-Deva" + ] + }, + { + "ndcg_at_1": 0.62615, + "ndcg_at_3": 0.72312, + "ndcg_at_5": 0.74842, + "ndcg_at_10": 0.76807, + "ndcg_at_20": 0.77722, + "ndcg_at_100": 0.78657, + "ndcg_at_1000": 0.78819, + "map_at_1": 0.603, + "map_at_3": 0.69493, + "map_at_5": 0.70994, + "map_at_10": 0.71856, + "map_at_20": 0.72155, + "map_at_100": 0.72305, + "map_at_1000": 0.72313, + "recall_at_1": 0.603, + "recall_at_3": 0.79026, + "recall_at_5": 0.84951, + "recall_at_10": 0.90807, + "recall_at_20": 0.94127, + "recall_at_100": 0.98842, + "recall_at_1000": 1.0, + "accuracy": 0.603, + "precision_at_1": 0.62615, + "precision_at_3": 0.27954, + "precision_at_5": 0.18121, + "precision_at_10": 0.09745, + "precision_at_20": 0.05102, + "precision_at_100": 0.01079, + "precision_at_1000": 0.00109, + "mrr_at_1": 0.626149, + "mrr_at_3": 0.709057, + "mrr_at_5": 0.722029, + "mrr_at_10": 0.729617, + "mrr_at_20": 0.73153, + "mrr_at_100": 0.732702, + "mrr_at_1000": 0.732766, + "nauc_ndcg_at_1_max": 0.358877, + "nauc_ndcg_at_1_std": -0.227364, + "nauc_ndcg_at_1_diff1": 0.709877, + "nauc_ndcg_at_3_max": 0.363217, + "nauc_ndcg_at_3_std": -0.289643, + "nauc_ndcg_at_3_diff1": 0.67885, + "nauc_ndcg_at_5_max": 0.377813, + "nauc_ndcg_at_5_std": -0.272705, + "nauc_ndcg_at_5_diff1": 0.674932, + "nauc_ndcg_at_10_max": 0.377594, + "nauc_ndcg_at_10_std": -0.2709, + "nauc_ndcg_at_10_diff1": 0.685033, + "nauc_ndcg_at_20_max": 0.381257, + "nauc_ndcg_at_20_std": -0.265273, + "nauc_ndcg_at_20_diff1": 0.685536, + "nauc_ndcg_at_100_max": 0.378416, + "nauc_ndcg_at_100_std": -0.257021, + "nauc_ndcg_at_100_diff1": 0.687697, + "nauc_ndcg_at_1000_max": 0.377206, + "nauc_ndcg_at_1000_std": -0.259374, + "nauc_ndcg_at_1000_diff1": 0.687983, + "nauc_map_at_1_max": 0.32349, + "nauc_map_at_1_std": -0.226504, + "nauc_map_at_1_diff1": 0.708454, + "nauc_map_at_3_max": 0.356433, + "nauc_map_at_3_std": -0.276642, + "nauc_map_at_3_diff1": 0.686871, + "nauc_map_at_5_max": 0.365342, + "nauc_map_at_5_std": -0.265772, + "nauc_map_at_5_diff1": 0.685198, + "nauc_map_at_10_max": 0.365614, + "nauc_map_at_10_std": -0.265439, + "nauc_map_at_10_diff1": 0.688594, + "nauc_map_at_20_max": 0.366946, + "nauc_map_at_20_std": -0.263406, + "nauc_map_at_20_diff1": 0.688815, + "nauc_map_at_100_max": 0.366751, + "nauc_map_at_100_std": -0.262162, + "nauc_map_at_100_diff1": 0.6893, + "nauc_map_at_1000_max": 0.366736, + "nauc_map_at_1000_std": -0.262212, + "nauc_map_at_1000_diff1": 0.689324, + "nauc_recall_at_1_max": 0.32349, + "nauc_recall_at_1_std": -0.226504, + "nauc_recall_at_1_diff1": 0.708454, + "nauc_recall_at_3_max": 0.360116, + "nauc_recall_at_3_std": -0.354112, + "nauc_recall_at_3_diff1": 0.645427, + "nauc_recall_at_5_max": 0.405125, + "nauc_recall_at_5_std": -0.318376, + "nauc_recall_at_5_diff1": 0.617656, + "nauc_recall_at_10_max": 0.40414, + "nauc_recall_at_10_std": -0.336225, + "nauc_recall_at_10_diff1": 0.646826, + "nauc_recall_at_20_max": 0.455001, + "nauc_recall_at_20_std": -0.326822, + "nauc_recall_at_20_diff1": 0.646997, + "nauc_recall_at_100_max": 0.531127, + "nauc_recall_at_100_std": 0.045983, + "nauc_recall_at_100_diff1": 0.656158, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.358877, + "nauc_precision_at_1_std": -0.227364, + "nauc_precision_at_1_diff1": 0.709877, + "nauc_precision_at_3_max": 0.362685, + "nauc_precision_at_3_std": -0.280333, + "nauc_precision_at_3_diff1": 0.477406, + "nauc_precision_at_5_max": 0.365889, + "nauc_precision_at_5_std": -0.195373, + "nauc_precision_at_5_diff1": 0.347552, + "nauc_precision_at_10_max": 0.315516, + "nauc_precision_at_10_std": -0.125315, + "nauc_precision_at_10_diff1": 0.22201, + "nauc_precision_at_20_max": 0.248008, + "nauc_precision_at_20_std": -0.01173, + "nauc_precision_at_20_diff1": 0.061097, + "nauc_precision_at_100_max": 0.127459, + "nauc_precision_at_100_std": 0.144027, + "nauc_precision_at_100_diff1": -0.146465, + "nauc_precision_at_1000_max": 0.089547, + "nauc_precision_at_1000_std": 0.140866, + "nauc_precision_at_1000_diff1": -0.201638, + "nauc_mrr_at_1_max": 0.358877, + "nauc_mrr_at_1_std": -0.227364, + "nauc_mrr_at_1_diff1": 0.709877, + "nauc_mrr_at_3_max": 0.375399, + "nauc_mrr_at_3_std": -0.260879, + "nauc_mrr_at_3_diff1": 0.684836, + "nauc_mrr_at_5_max": 0.380002, + "nauc_mrr_at_5_std": -0.253356, + "nauc_mrr_at_5_diff1": 0.682841, + "nauc_mrr_at_10_max": 0.380328, + "nauc_mrr_at_10_std": -0.250571, + "nauc_mrr_at_10_diff1": 0.687702, + "nauc_mrr_at_20_max": 0.380638, + "nauc_mrr_at_20_std": -0.249977, + "nauc_mrr_at_20_diff1": 0.687373, + "nauc_mrr_at_100_max": 0.380003, + "nauc_mrr_at_100_std": -0.249633, + "nauc_mrr_at_100_diff1": 0.687671, + "nauc_mrr_at_1000_max": 0.379975, + "nauc_mrr_at_1000_std": -0.249612, + "nauc_mrr_at_1000_diff1": 0.687674, + "hit_rate_at_1": 0.62615, + "hit_rate_at_3": 0.80797, + "hit_rate_at_5": 0.86415, + "hit_rate_at_10": 0.92135, + "hit_rate_at_20": 0.94791, + "hit_rate_at_100": 0.98979, + "hit_rate_at_1000": 1.0, + "main_score": 0.76807, + "hf_subset": "ur", + "languages": [ + "urd-Arab" + ] + } + ] + }, + "evaluation_time": 196.8238456249237, + "kg_co2_emissions": null, + "date": 1777840811.864055 +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/LexRetrieval.v1.json b/results/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/LexRetrieval.v1.json new file mode 100644 index 0000000000..2f7efd4337 --- /dev/null +++ b/results/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/LexRetrieval.v1.json @@ -0,0 +1,167 @@ +{ + "dataset_revision": "b5b22dee3a30bb32ac579c14afe9eb152e8c70c7", + "task_name": "LexRetrieval.v1", + "mteb_version": "2.12.38", + "scores": { + "test": [ + { + "ndcg_at_1": 0.89164, + "ndcg_at_3": 0.9204, + "ndcg_at_5": 0.92552, + "ndcg_at_10": 0.92927, + "ndcg_at_20": 0.93165, + "ndcg_at_100": 0.93454, + "ndcg_at_1000": 0.93581, + "map_at_1": 0.89164, + "map_at_3": 0.91365, + "map_at_5": 0.9165, + "map_at_10": 0.91807, + "map_at_20": 0.91872, + "map_at_100": 0.91913, + "map_at_1000": 0.91918, + "recall_at_1": 0.89164, + "recall_at_3": 0.93979, + "recall_at_5": 0.95217, + "recall_at_10": 0.9637, + "recall_at_20": 0.9731, + "recall_at_100": 0.98858, + "recall_at_1000": 0.99851, + "accuracy": 0.89164, + "precision_at_1": 0.89164, + "precision_at_3": 0.31326, + "precision_at_5": 0.19043, + "precision_at_10": 0.09637, + "precision_at_20": 0.04865, + "precision_at_100": 0.00989, + "precision_at_1000": 0.001, + "mrr_at_1": 0.891641, + "mrr_at_3": 0.913651, + "mrr_at_5": 0.916501, + "mrr_at_10": 0.918067, + "mrr_at_20": 0.918721, + "mrr_at_100": 0.919126, + "mrr_at_1000": 0.919176, + "nauc_ndcg_at_1_max": 0.782862, + "nauc_ndcg_at_1_std": 0.103814, + "nauc_ndcg_at_1_diff1": 0.907851, + "nauc_ndcg_at_3_max": 0.822413, + "nauc_ndcg_at_3_std": 0.133043, + "nauc_ndcg_at_3_diff1": 0.911819, + "nauc_ndcg_at_5_max": 0.822885, + "nauc_ndcg_at_5_std": 0.139543, + "nauc_ndcg_at_5_diff1": 0.911629, + "nauc_ndcg_at_10_max": 0.820739, + "nauc_ndcg_at_10_std": 0.141836, + "nauc_ndcg_at_10_diff1": 0.910722, + "nauc_ndcg_at_20_max": 0.818667, + "nauc_ndcg_at_20_std": 0.140833, + "nauc_ndcg_at_20_diff1": 0.909929, + "nauc_ndcg_at_100_max": 0.817021, + "nauc_ndcg_at_100_std": 0.141941, + "nauc_ndcg_at_100_diff1": 0.910076, + "nauc_ndcg_at_1000_max": 0.814936, + "nauc_ndcg_at_1000_std": 0.136234, + "nauc_ndcg_at_1000_diff1": 0.909986, + "nauc_map_at_1_max": 0.782862, + "nauc_map_at_1_std": 0.103814, + "nauc_map_at_1_diff1": 0.907851, + "nauc_map_at_3_max": 0.811637, + "nauc_map_at_3_std": 0.125031, + "nauc_map_at_3_diff1": 0.910321, + "nauc_map_at_5_max": 0.811465, + "nauc_map_at_5_std": 0.127974, + "nauc_map_at_5_diff1": 0.910143, + "nauc_map_at_10_max": 0.810523, + "nauc_map_at_10_std": 0.128622, + "nauc_map_at_10_diff1": 0.909794, + "nauc_map_at_20_max": 0.809964, + "nauc_map_at_20_std": 0.128388, + "nauc_map_at_20_diff1": 0.909585, + "nauc_map_at_100_max": 0.809724, + "nauc_map_at_100_std": 0.128621, + "nauc_map_at_100_diff1": 0.909599, + "nauc_map_at_1000_max": 0.809655, + "nauc_map_at_1000_std": 0.128459, + "nauc_map_at_1000_diff1": 0.909592, + "nauc_recall_at_1_max": 0.782862, + "nauc_recall_at_1_std": 0.103814, + "nauc_recall_at_1_diff1": 0.907851, + "nauc_recall_at_3_max": 0.866053, + "nauc_recall_at_3_std": 0.165521, + "nauc_recall_at_3_diff1": 0.918155, + "nauc_recall_at_5_max": 0.879384, + "nauc_recall_at_5_std": 0.19839, + "nauc_recall_at_5_diff1": 0.919315, + "nauc_recall_at_10_max": 0.883975, + "nauc_recall_at_10_std": 0.230205, + "nauc_recall_at_10_diff1": 0.916279, + "nauc_recall_at_20_max": 0.885288, + "nauc_recall_at_20_std": 0.248529, + "nauc_recall_at_20_diff1": 0.910702, + "nauc_recall_at_100_max": 0.925743, + "nauc_recall_at_100_std": 0.414102, + "nauc_recall_at_100_diff1": 0.916013, + "nauc_recall_at_1000_max": 0.952147, + "nauc_recall_at_1000_std": 0.266588, + "nauc_recall_at_1000_diff1": 0.933472, + "nauc_precision_at_1_max": 0.782862, + "nauc_precision_at_1_std": 0.103814, + "nauc_precision_at_1_diff1": 0.907851, + "nauc_precision_at_3_max": 0.866053, + "nauc_precision_at_3_std": 0.165521, + "nauc_precision_at_3_diff1": 0.918155, + "nauc_precision_at_5_max": 0.879384, + "nauc_precision_at_5_std": 0.19839, + "nauc_precision_at_5_diff1": 0.919315, + "nauc_precision_at_10_max": 0.883975, + "nauc_precision_at_10_std": 0.230205, + "nauc_precision_at_10_diff1": 0.916279, + "nauc_precision_at_20_max": 0.885288, + "nauc_precision_at_20_std": 0.248529, + "nauc_precision_at_20_diff1": 0.910702, + "nauc_precision_at_100_max": 0.925743, + "nauc_precision_at_100_std": 0.414102, + "nauc_precision_at_100_diff1": 0.916013, + "nauc_precision_at_1000_max": 0.952147, + "nauc_precision_at_1000_std": 0.266588, + "nauc_precision_at_1000_diff1": 0.933472, + "nauc_mrr_at_1_max": 0.782862, + "nauc_mrr_at_1_std": 0.103814, + "nauc_mrr_at_1_diff1": 0.907851, + "nauc_mrr_at_3_max": 0.811637, + "nauc_mrr_at_3_std": 0.125031, + "nauc_mrr_at_3_diff1": 0.910321, + "nauc_mrr_at_5_max": 0.811465, + "nauc_mrr_at_5_std": 0.127974, + "nauc_mrr_at_5_diff1": 0.910143, + "nauc_mrr_at_10_max": 0.810523, + "nauc_mrr_at_10_std": 0.128622, + "nauc_mrr_at_10_diff1": 0.909794, + "nauc_mrr_at_20_max": 0.809964, + "nauc_mrr_at_20_std": 0.128388, + "nauc_mrr_at_20_diff1": 0.909585, + "nauc_mrr_at_100_max": 0.809724, + "nauc_mrr_at_100_std": 0.128621, + "nauc_mrr_at_100_diff1": 0.909599, + "nauc_mrr_at_1000_max": 0.809655, + "nauc_mrr_at_1000_std": 0.128459, + "nauc_mrr_at_1000_diff1": 0.909592, + "hit_rate_at_1": 0.89164, + "hit_rate_at_3": 0.93979, + "hit_rate_at_5": 0.95217, + "hit_rate_at_10": 0.9637, + "hit_rate_at_20": 0.9731, + "hit_rate_at_100": 0.98858, + "hit_rate_at_1000": 0.99851, + "main_score": 0.92927, + "hf_subset": "default", + "languages": [ + "dan-Latn" + ] + } + ] + }, + "evaluation_time": 87.40175318717957, + "kg_co2_emissions": null, + "date": 1777840582.780445 +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/PublicNewsRetrieval.json b/results/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/PublicNewsRetrieval.json new file mode 100644 index 0000000000..366301d180 --- /dev/null +++ b/results/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/PublicNewsRetrieval.json @@ -0,0 +1,322 @@ +{ + "dataset_revision": "82dad8c469646feae9631d69ba92bebfc0a586e9", + "task_name": "PublicNewsRetrieval", + "mteb_version": "2.12.37", + "scores": { + "test": [ + { + "ndcg_at_1": 0.78485, + "ndcg_at_3": 0.83478, + "ndcg_at_5": 0.8463, + "ndcg_at_10": 0.85673, + "ndcg_at_20": 0.86334, + "ndcg_at_100": 0.86922, + "ndcg_at_1000": 0.87134, + "map_at_1": 0.77441, + "map_at_3": 0.82031, + "map_at_5": 0.82748, + "map_at_10": 0.83199, + "map_at_20": 0.83398, + "map_at_100": 0.83484, + "map_at_1000": 0.83493, + "recall_at_1": 0.77441, + "recall_at_3": 0.86975, + "recall_at_5": 0.89662, + "recall_at_10": 0.92756, + "recall_at_20": 0.95241, + "recall_at_100": 0.98336, + "recall_at_1000": 0.99899, + "accuracy": 0.77441, + "precision_at_1": 0.78485, + "precision_at_3": 0.29692, + "precision_at_5": 0.18458, + "precision_at_10": 0.09605, + "precision_at_20": 0.04956, + "precision_at_100": 0.01027, + "precision_at_1000": 0.00105, + "mrr_at_1": 0.784853, + "mrr_at_3": 0.826631, + "mrr_at_5": 0.832261, + "mrr_at_10": 0.836374, + "mrr_at_20": 0.838046, + "mrr_at_100": 0.838759, + "mrr_at_1000": 0.838838, + "nauc_ndcg_at_1_max": 0.435822, + "nauc_ndcg_at_1_std": -0.531603, + "nauc_ndcg_at_1_diff1": 0.892289, + "nauc_ndcg_at_3_max": 0.44717, + "nauc_ndcg_at_3_std": -0.579379, + "nauc_ndcg_at_3_diff1": 0.874739, + "nauc_ndcg_at_5_max": 0.459196, + "nauc_ndcg_at_5_std": -0.559499, + "nauc_ndcg_at_5_diff1": 0.87059, + "nauc_ndcg_at_10_max": 0.477133, + "nauc_ndcg_at_10_std": -0.541695, + "nauc_ndcg_at_10_diff1": 0.875575, + "nauc_ndcg_at_20_max": 0.480121, + "nauc_ndcg_at_20_std": -0.526619, + "nauc_ndcg_at_20_diff1": 0.878735, + "nauc_ndcg_at_100_max": 0.470169, + "nauc_ndcg_at_100_std": -0.52875, + "nauc_ndcg_at_100_diff1": 0.878955, + "nauc_ndcg_at_1000_max": 0.465287, + "nauc_ndcg_at_1000_std": -0.537324, + "nauc_ndcg_at_1000_diff1": 0.878846, + "nauc_map_at_1_max": 0.415192, + "nauc_map_at_1_std": -0.530811, + "nauc_map_at_1_diff1": 0.890549, + "nauc_map_at_3_max": 0.440402, + "nauc_map_at_3_std": -0.568469, + "nauc_map_at_3_diff1": 0.879772, + "nauc_map_at_5_max": 0.447229, + "nauc_map_at_5_std": -0.557152, + "nauc_map_at_5_diff1": 0.877432, + "nauc_map_at_10_max": 0.454456, + "nauc_map_at_10_std": -0.549811, + "nauc_map_at_10_diff1": 0.879302, + "nauc_map_at_20_max": 0.454983, + "nauc_map_at_20_std": -0.546181, + "nauc_map_at_20_diff1": 0.880058, + "nauc_map_at_100_max": 0.453885, + "nauc_map_at_100_std": -0.545997, + "nauc_map_at_100_diff1": 0.880095, + "nauc_map_at_1000_max": 0.453696, + "nauc_map_at_1000_std": -0.546264, + "nauc_map_at_1000_diff1": 0.880075, + "nauc_recall_at_1_max": 0.415192, + "nauc_recall_at_1_std": -0.530811, + "nauc_recall_at_1_diff1": 0.890549, + "nauc_recall_at_3_max": 0.460184, + "nauc_recall_at_3_std": -0.627774, + "nauc_recall_at_3_diff1": 0.852732, + "nauc_recall_at_5_max": 0.500345, + "nauc_recall_at_5_std": -0.573167, + "nauc_recall_at_5_diff1": 0.835692, + "nauc_recall_at_10_max": 0.604588, + "nauc_recall_at_10_std": -0.491627, + "nauc_recall_at_10_diff1": 0.845089, + "nauc_recall_at_20_max": 0.702412, + "nauc_recall_at_20_std": -0.316669, + "nauc_recall_at_20_diff1": 0.866956, + "nauc_recall_at_100_max": 0.718065, + "nauc_recall_at_100_std": -0.077076, + "nauc_recall_at_100_diff1": 0.861716, + "nauc_recall_at_1000_max": 0.488785, + "nauc_recall_at_1000_std": -0.759424, + "nauc_recall_at_1000_diff1": 0.814861, + "nauc_precision_at_1_max": 0.435822, + "nauc_precision_at_1_std": -0.531603, + "nauc_precision_at_1_diff1": 0.892289, + "nauc_precision_at_3_max": 0.418776, + "nauc_precision_at_3_std": -0.493666, + "nauc_precision_at_3_diff1": 0.654103, + "nauc_precision_at_5_max": 0.392672, + "nauc_precision_at_5_std": -0.335468, + "nauc_precision_at_5_diff1": 0.49759, + "nauc_precision_at_10_max": 0.409121, + "nauc_precision_at_10_std": -0.114563, + "nauc_precision_at_10_diff1": 0.324954, + "nauc_precision_at_20_max": 0.34235, + "nauc_precision_at_20_std": 0.119853, + "nauc_precision_at_20_diff1": 0.128747, + "nauc_precision_at_100_max": 0.143385, + "nauc_precision_at_100_std": 0.377691, + "nauc_precision_at_100_diff1": -0.194237, + "nauc_precision_at_1000_max": -0.011137, + "nauc_precision_at_1000_std": 0.398437, + "nauc_precision_at_1000_diff1": -0.40264, + "nauc_mrr_at_1_max": 0.435822, + "nauc_mrr_at_1_std": -0.531603, + "nauc_mrr_at_1_diff1": 0.892289, + "nauc_mrr_at_3_max": 0.451335, + "nauc_mrr_at_3_std": -0.562479, + "nauc_mrr_at_3_diff1": 0.880162, + "nauc_mrr_at_5_max": 0.45479, + "nauc_mrr_at_5_std": -0.552362, + "nauc_mrr_at_5_diff1": 0.878468, + "nauc_mrr_at_10_max": 0.459962, + "nauc_mrr_at_10_std": -0.54572, + "nauc_mrr_at_10_diff1": 0.880855, + "nauc_mrr_at_20_max": 0.460029, + "nauc_mrr_at_20_std": -0.542915, + "nauc_mrr_at_20_diff1": 0.881645, + "nauc_mrr_at_100_max": 0.458728, + "nauc_mrr_at_100_std": -0.543601, + "nauc_mrr_at_100_diff1": 0.881579, + "nauc_mrr_at_1000_max": 0.458546, + "nauc_mrr_at_1000_std": -0.543838, + "nauc_mrr_at_1000_diff1": 0.881553, + "hit_rate_at_1": 0.78485, + "hit_rate_at_3": 0.87735, + "hit_rate_at_5": 0.90147, + "hit_rate_at_10": 0.93231, + "hit_rate_at_20": 0.95643, + "hit_rate_at_100": 0.98525, + "hit_rate_at_1000": 0.99933, + "main_score": 0.85673, + "hf_subset": "pl", + "languages": [ + "pol-Latn" + ] + }, + { + "ndcg_at_1": 0.57825, + "ndcg_at_3": 0.66601, + "ndcg_at_5": 0.68825, + "ndcg_at_10": 0.70925, + "ndcg_at_20": 0.72263, + "ndcg_at_100": 0.73681, + "ndcg_at_1000": 0.74102, + "map_at_1": 0.56391, + "map_at_3": 0.64167, + "map_at_5": 0.65473, + "map_at_10": 0.66404, + "map_at_20": 0.66788, + "map_at_100": 0.66995, + "map_at_1000": 0.67014, + "recall_at_1": 0.56391, + "recall_at_3": 0.72821, + "recall_at_5": 0.78061, + "recall_at_10": 0.84318, + "recall_at_20": 0.89511, + "recall_at_100": 0.96887, + "recall_at_1000": 0.99976, + "accuracy": 0.56391, + "precision_at_1": 0.57825, + "precision_at_3": 0.25469, + "precision_at_5": 0.16492, + "precision_at_10": 0.08955, + "precision_at_20": 0.04759, + "precision_at_100": 0.01035, + "precision_at_1000": 0.00107, + "mrr_at_1": 0.578251, + "mrr_at_3": 0.649251, + "mrr_at_5": 0.66067, + "mrr_at_10": 0.668841, + "mrr_at_20": 0.672347, + "mrr_at_100": 0.674275, + "mrr_at_1000": 0.674439, + "nauc_ndcg_at_1_max": 0.168216, + "nauc_ndcg_at_1_std": -0.087738, + "nauc_ndcg_at_1_diff1": 0.737266, + "nauc_ndcg_at_3_max": 0.165422, + "nauc_ndcg_at_3_std": -0.119352, + "nauc_ndcg_at_3_diff1": 0.705964, + "nauc_ndcg_at_5_max": 0.161653, + "nauc_ndcg_at_5_std": -0.127728, + "nauc_ndcg_at_5_diff1": 0.706682, + "nauc_ndcg_at_10_max": 0.158143, + "nauc_ndcg_at_10_std": -0.122216, + "nauc_ndcg_at_10_diff1": 0.708585, + "nauc_ndcg_at_20_max": 0.160945, + "nauc_ndcg_at_20_std": -0.118892, + "nauc_ndcg_at_20_diff1": 0.706751, + "nauc_ndcg_at_100_max": 0.165065, + "nauc_ndcg_at_100_std": -0.113865, + "nauc_ndcg_at_100_diff1": 0.711544, + "nauc_ndcg_at_1000_max": 0.165426, + "nauc_ndcg_at_1000_std": -0.112665, + "nauc_ndcg_at_1000_diff1": 0.71234, + "nauc_map_at_1_max": 0.151109, + "nauc_map_at_1_std": -0.085662, + "nauc_map_at_1_diff1": 0.734539, + "nauc_map_at_3_max": 0.163662, + "nauc_map_at_3_std": -0.110765, + "nauc_map_at_3_diff1": 0.714198, + "nauc_map_at_5_max": 0.16302, + "nauc_map_at_5_std": -0.115518, + "nauc_map_at_5_diff1": 0.715685, + "nauc_map_at_10_max": 0.16243, + "nauc_map_at_10_std": -0.112768, + "nauc_map_at_10_diff1": 0.716707, + "nauc_map_at_20_max": 0.16372, + "nauc_map_at_20_std": -0.111443, + "nauc_map_at_20_diff1": 0.716281, + "nauc_map_at_100_max": 0.16428, + "nauc_map_at_100_std": -0.110829, + "nauc_map_at_100_diff1": 0.716883, + "nauc_map_at_1000_max": 0.164294, + "nauc_map_at_1000_std": -0.110731, + "nauc_map_at_1000_diff1": 0.716907, + "nauc_recall_at_1_max": 0.151109, + "nauc_recall_at_1_std": -0.085662, + "nauc_recall_at_1_diff1": 0.734539, + "nauc_recall_at_3_max": 0.160209, + "nauc_recall_at_3_std": -0.148154, + "nauc_recall_at_3_diff1": 0.672347, + "nauc_recall_at_5_max": 0.14551, + "nauc_recall_at_5_std": -0.179734, + "nauc_recall_at_5_diff1": 0.669974, + "nauc_recall_at_10_max": 0.120586, + "nauc_recall_at_10_std": -0.175864, + "nauc_recall_at_10_diff1": 0.666141, + "nauc_recall_at_20_max": 0.123286, + "nauc_recall_at_20_std": -0.176675, + "nauc_recall_at_20_diff1": 0.6367, + "nauc_recall_at_100_max": 0.159458, + "nauc_recall_at_100_std": -0.147975, + "nauc_recall_at_100_diff1": 0.659013, + "nauc_recall_at_1000_max": 0.869247, + "nauc_recall_at_1000_std": -1.741868, + "nauc_recall_at_1000_diff1": 0.357985, + "nauc_precision_at_1_max": 0.168216, + "nauc_precision_at_1_std": -0.087738, + "nauc_precision_at_1_diff1": 0.737266, + "nauc_precision_at_3_max": 0.186842, + "nauc_precision_at_3_std": -0.123102, + "nauc_precision_at_3_diff1": 0.553679, + "nauc_precision_at_5_max": 0.172182, + "nauc_precision_at_5_std": -0.132407, + "nauc_precision_at_5_diff1": 0.481232, + "nauc_precision_at_10_max": 0.142346, + "nauc_precision_at_10_std": -0.094325, + "nauc_precision_at_10_diff1": 0.372247, + "nauc_precision_at_20_max": 0.147253, + "nauc_precision_at_20_std": -0.055084, + "nauc_precision_at_20_diff1": 0.251401, + "nauc_precision_at_100_max": 0.150677, + "nauc_precision_at_100_std": 0.059252, + "nauc_precision_at_100_diff1": -0.033517, + "nauc_precision_at_1000_max": 0.135873, + "nauc_precision_at_1000_std": 0.135825, + "nauc_precision_at_1000_diff1": -0.232164, + "nauc_mrr_at_1_max": 0.168216, + "nauc_mrr_at_1_std": -0.087738, + "nauc_mrr_at_1_diff1": 0.737266, + "nauc_mrr_at_3_max": 0.17049, + "nauc_mrr_at_3_std": -0.111682, + "nauc_mrr_at_3_diff1": 0.714038, + "nauc_mrr_at_5_max": 0.166767, + "nauc_mrr_at_5_std": -0.114367, + "nauc_mrr_at_5_diff1": 0.713895, + "nauc_mrr_at_10_max": 0.165202, + "nauc_mrr_at_10_std": -0.11169, + "nauc_mrr_at_10_diff1": 0.715088, + "nauc_mrr_at_20_max": 0.16543, + "nauc_mrr_at_20_std": -0.111144, + "nauc_mrr_at_20_diff1": 0.71468, + "nauc_mrr_at_100_max": 0.166122, + "nauc_mrr_at_100_std": -0.110552, + "nauc_mrr_at_100_diff1": 0.715318, + "nauc_mrr_at_1000_max": 0.166135, + "nauc_mrr_at_1000_std": -0.110519, + "nauc_mrr_at_1000_diff1": 0.715348, + "hit_rate_at_1": 0.57825, + "hit_rate_at_3": 0.73853, + "hit_rate_at_5": 0.78913, + "hit_rate_at_10": 0.84917, + "hit_rate_at_20": 0.89976, + "hit_rate_at_100": 0.97163, + "hit_rate_at_1000": 1.0, + "main_score": 0.70925, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ] + } + ] + }, + "evaluation_time": 168.65391182899475, + "kg_co2_emissions": null, + "date": 1777964531.864433 +} \ No newline at end of file diff --git a/results/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/model_meta.json b/results/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/model_meta.json index bfc2d2c6b1..da9f9b3d1a 100644 --- a/results/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/model_meta.json +++ b/results/intfloat__multilingual-e5-large/ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb/model_meta.json @@ -1 +1,171 @@ -{"name": "intfloat/multilingual-e5-large", "revision": "ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb", "release_date": "2024-02-08", "languages": ["afr-Latn", "amh-Latn", "ara-Latn", "asm-Latn", "aze-Latn", "bel-Latn", "bul-Latn", "ben-Latn", "ben-Beng", "bre-Latn", "bos-Latn", "cat-Latn", "ces-Latn", "cym-Latn", "dan-Latn", "deu-Latn", "ell-Latn", "eng-Latn", "epo-Latn", "spa-Latn", "est-Latn", "eus-Latn", "fas-Latn", "fin-Latn", "fra-Latn", "fry-Latn", "gle-Latn", "gla-Latn", "glg-Latn", "guj-Latn", "hau-Latn", "heb-Latn", "hin-Latn", "hin-Deva", "hrv-Latn", "hun-Latn", "hye-Latn", "ind-Latn", "isl-Latn", "ita-Latn", "jpn-Latn", "jav-Latn", "kat-Latn", "kaz-Latn", "khm-Latn", "kan-Latn", "kor-Latn", "kur-Latn", "kir-Latn", "lat-Latn", "lao-Latn", "lit-Latn", "lav-Latn", "mlg-Latn", "mkd-Latn", "mal-Latn", "mon-Latn", "mar-Latn", "msa-Latn", "mya-Latn", "nep-Latn", "nld-Latn", "nob-Latn", "orm-Latn", "ori-Latn", "pan-Latn", "pol-Latn", "pus-Latn", "por-Latn", "ron-Latn", "rus-Latn", "san-Latn", "snd-Latn", "sin-Latn", "slk-Latn", "slv-Latn", "som-Latn", "sqi-Latn", "srp-Latn", "sun-Latn", "swe-Latn", "swa-Latn", "tam-Latn", "tam-Taml", "tel-Latn", "tel-Telu", "tha-Latn", "tgl-Latn", "tur-Latn", "uig-Latn", "ukr-Latn", "urd-Latn", "urd-Arab", "uzb-Latn", "vie-Latn", "xho-Latn", "yid-Latn", "zho-Hant", "zho-Hans"], "n_parameters": 560000000, "memory_usage_mb": 2136.0, "max_tokens": 514.0, "embed_dim": 1024, "license": "mit", "open_weights": true, "public_training_code": null, "public_training_data": null, "framework": ["Sentence Transformers", "PyTorch"], "reference": "https://huggingface.co/intfloat/multilingual-e5-large", "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": {"MSMARCO": ["train"], "MSMARCOHardNegatives": ["train"], "NanoMSMARCORetrieval": ["train"], "MSMARCO-PL": ["train"], "mMARCO-NL": ["train"], "NQ": ["train"], "NQHardNegatives": ["train"], "NanoNQRetrieval": ["train"], "NQ-PL": ["train"], "NQ-NL": ["train"], "XQuADRetrieval": ["train"], "FEVER": ["train"], "FEVERHardNegatives": ["train"], "FEVER-NL": ["train"], "HotpotQA": ["train"], "HotpotQAHardNegatives": ["train"], "HotpotQA-PL": ["train"], "HotpotQA-NL": ["train"], "MIRACLRetrieval": ["train"], "MIRACLRetrievalHardNegatives": ["train"], "MIRACLReranking": ["train"], "MrTidyRetrieval": ["train"]}, "adapted_from": "FacebookAI/xlm-roberta-large", "superseded_by": null, "is_cross_encoder": null, "modalities": ["text"], "loader": "sentence_transformers_loader"} \ No newline at end of file +{ + "loader_kwargs": { + "model_prompts": { + "query": "query: ", + "document": "passage: " + } + }, + "name": "intfloat/multilingual-e5-large", + "revision": "ab10c1a7f42e74530fe7ae5be82e6d4f11a719eb", + "release_date": "2024-02-08", + "languages": [ + "afr-Latn", + "amh-Latn", + "ara-Latn", + "asm-Latn", + "aze-Latn", + "bel-Latn", + "bul-Latn", + "ben-Latn", + "ben-Beng", + "bre-Latn", + "bos-Latn", + "cat-Latn", + "ces-Latn", + "cym-Latn", + "dan-Latn", + "deu-Latn", + "ell-Latn", + "eng-Latn", + "epo-Latn", + "spa-Latn", + "est-Latn", + "eus-Latn", + "fas-Latn", + "fin-Latn", + "fra-Latn", + "fry-Latn", + "gle-Latn", + "gla-Latn", + "glg-Latn", + "guj-Latn", + "hau-Latn", + "heb-Latn", + "hin-Latn", + "hin-Deva", + "hrv-Latn", + "hun-Latn", + "hye-Latn", + "ind-Latn", + "isl-Latn", + "ita-Latn", + "jpn-Latn", + "jav-Latn", + "kat-Latn", + "kaz-Latn", + "khm-Latn", + "kan-Latn", + "kor-Latn", + "kur-Latn", + "kir-Latn", + "lat-Latn", + "lao-Latn", + "lit-Latn", + "lav-Latn", + "mlg-Latn", + "mkd-Latn", + "mal-Latn", + "mon-Latn", + "mar-Latn", + "msa-Latn", + "mya-Latn", + "nep-Latn", + "nld-Latn", + "nob-Latn", + "orm-Latn", + "ori-Latn", + "pan-Latn", + "pol-Latn", + "pus-Latn", + "por-Latn", + "ron-Latn", + "rus-Latn", + "san-Latn", + "snd-Latn", + "sin-Latn", + "slk-Latn", + "slv-Latn", + "som-Latn", + "sqi-Latn", + "srp-Latn", + "sun-Latn", + "swe-Latn", + "swa-Latn", + "tam-Latn", + "tam-Taml", + "tel-Latn", + "tel-Telu", + "tha-Latn", + "tgl-Latn", + "tur-Latn", + "uig-Latn", + "ukr-Latn", + "urd-Latn", + "urd-Arab", + "uzb-Latn", + "vie-Latn", + "xho-Latn", + "yid-Latn", + "zho-Hant", + "zho-Hans" + ], + "n_parameters": 560000000, + "n_active_parameters_override": null, + "n_embedding_parameters": 256002048, + "memory_usage_mb": 2136.0, + "max_tokens": 514.0, + "embed_dim": 1024, + "license": "mit", + "open_weights": true, + "public_training_code": null, + "public_training_data": null, + "framework": [ + "Sentence Transformers", + "PyTorch", + "ONNX", + "safetensors" + ], + "reference": "https://huggingface.co/intfloat/multilingual-e5-large", + "similarity_fn_name": "cosine", + "use_instructions": true, + "training_datasets": [ + "MIRACLRetrievalHardNegatives", + "MIRACLRetrieval", + "NQ-PL", + "NQ", + "HotpotQA-PL", + "MSMARCO", + "NQHardNegatives", + "HotpotQA", + "MIRACLReranking", + "NQ-NL", + "HotpotQA-NL", + "XQuADRetrieval", + "NanoNQRetrieval", + "FEVER", + "FEVER-PL", + "MSMARCO-PL", + "mMARCO-NL", + "FEVERHardNegatives", + "HotpotQAHardNegatives", + "MrTidyRetrieval", + "MSMARCOHardNegatives", + "NanoMSMARCORetrieval", + "FEVER-NL" + ], + "adapted_from": "FacebookAI/xlm-roberta-large", + "superseded_by": null, + "modalities": [ + "text" + ], + "model_type": [ + "dense" + ], + "citation": "\n@article{wang2024multilingual,\n title={Multilingual E5 Text Embeddings: A Technical Report},\n author={Wang, Liang and Yang, Nan and Huang, Xiaolong and Yang, Linjun and Majumder, Rangan and Wei, Furu},\n journal={arXiv preprint arXiv:2402.05672},\n year={2024}\n}\n", + "contacts": null, + "experiment_kwargs": null, + "output_dtypes": null, + "extra_requirements_groups": null, + "loader": "SentenceTransformerEncoderWrapper", + "is_cross_encoder": false +} \ No newline at end of file diff --git a/results/jinaai__jina-embeddings-v5-text-small/dd76d535f5447ca3897a9c893fb1e612ead98192/ElasticKBRetrieval.json b/results/jinaai__jina-embeddings-v5-text-small/dd76d535f5447ca3897a9c893fb1e612ead98192/ElasticKBRetrieval.json new file mode 100644 index 0000000000..3db720a09f --- /dev/null +++ b/results/jinaai__jina-embeddings-v5-text-small/dd76d535f5447ca3897a9c893fb1e612ead98192/ElasticKBRetrieval.json @@ -0,0 +1,324 @@ +{ + "dataset_revision": "21bdf1e024bf7c9f46720017559ce2f8c6116507", + "task_name": "ElasticKBRetrieval", + "mteb_version": "2.12.37", + "scores": { + "synthetic_test": [ + { + "ndcg_at_1": 0.83511, + "ndcg_at_3": 0.64712, + "ndcg_at_5": 0.59965, + "ndcg_at_10": 0.57179, + "ndcg_at_20": 0.5694, + "ndcg_at_100": 0.61283, + "ndcg_at_1000": 0.67636, + "map_at_1": 0.27043, + "map_at_3": 0.3315, + "map_at_5": 0.3481, + "map_at_10": 0.37355, + "map_at_20": 0.39835, + "map_at_100": 0.42837, + "map_at_1000": 0.44077, + "recall_at_1": 0.27043, + "recall_at_3": 0.34659, + "recall_at_5": 0.37579, + "recall_at_10": 0.43147, + "recall_at_20": 0.50544, + "recall_at_100": 0.67811, + "recall_at_1000": 0.87276, + "accuracy": 0.27043, + "precision_at_1": 0.83511, + "precision_at_3": 0.47872, + "precision_at_5": 0.36383, + "precision_at_10": 0.25798, + "precision_at_20": 0.1859, + "precision_at_100": 0.07005, + "precision_at_1000": 0.01301, + "mrr_at_1": 0.835106, + "mrr_at_3": 0.882979, + "mrr_at_5": 0.887766, + "mrr_at_10": 0.890071, + "mrr_at_20": 0.890554, + "mrr_at_100": 0.891136, + "mrr_at_1000": 0.891136, + "nauc_ndcg_at_1_max": 0.305281, + "nauc_ndcg_at_1_std": -0.102015, + "nauc_ndcg_at_1_diff1": 0.540248, + "nauc_ndcg_at_3_max": 0.176921, + "nauc_ndcg_at_3_std": -0.015337, + "nauc_ndcg_at_3_diff1": 0.167976, + "nauc_ndcg_at_5_max": 0.179874, + "nauc_ndcg_at_5_std": 0.010608, + "nauc_ndcg_at_5_diff1": 0.182217, + "nauc_ndcg_at_10_max": 0.236472, + "nauc_ndcg_at_10_std": 0.053123, + "nauc_ndcg_at_10_diff1": 0.258287, + "nauc_ndcg_at_20_max": 0.218735, + "nauc_ndcg_at_20_std": 0.043185, + "nauc_ndcg_at_20_diff1": 0.286078, + "nauc_ndcg_at_100_max": 0.278608, + "nauc_ndcg_at_100_std": 0.111001, + "nauc_ndcg_at_100_diff1": 0.300066, + "nauc_ndcg_at_1000_max": 0.256161, + "nauc_ndcg_at_1000_std": 0.153797, + "nauc_ndcg_at_1000_diff1": 0.250715, + "nauc_map_at_1_max": 0.169259, + "nauc_map_at_1_std": -0.15201, + "nauc_map_at_1_diff1": 0.476628, + "nauc_map_at_3_max": 0.153285, + "nauc_map_at_3_std": -0.164878, + "nauc_map_at_3_diff1": 0.398342, + "nauc_map_at_5_max": 0.162568, + "nauc_map_at_5_std": -0.148554, + "nauc_map_at_5_diff1": 0.384327, + "nauc_map_at_10_max": 0.199345, + "nauc_map_at_10_std": -0.102045, + "nauc_map_at_10_diff1": 0.370503, + "nauc_map_at_20_max": 0.210635, + "nauc_map_at_20_std": -0.049857, + "nauc_map_at_20_diff1": 0.335471, + "nauc_map_at_100_max": 0.226642, + "nauc_map_at_100_std": 0.010015, + "nauc_map_at_100_diff1": 0.315652, + "nauc_map_at_1000_max": 0.220366, + "nauc_map_at_1000_std": 0.021748, + "nauc_map_at_1000_diff1": 0.306225, + "nauc_recall_at_1_max": 0.169259, + "nauc_recall_at_1_std": -0.15201, + "nauc_recall_at_1_diff1": 0.476628, + "nauc_recall_at_3_max": 0.137464, + "nauc_recall_at_3_std": -0.17316, + "nauc_recall_at_3_diff1": 0.378429, + "nauc_recall_at_5_max": 0.15, + "nauc_recall_at_5_std": -0.150243, + "nauc_recall_at_5_diff1": 0.35463, + "nauc_recall_at_10_max": 0.227231, + "nauc_recall_at_10_std": -0.068627, + "nauc_recall_at_10_diff1": 0.348743, + "nauc_recall_at_20_max": 0.216257, + "nauc_recall_at_20_std": 0.000315, + "nauc_recall_at_20_diff1": 0.269933, + "nauc_recall_at_100_max": 0.265584, + "nauc_recall_at_100_std": 0.189137, + "nauc_recall_at_100_diff1": 0.268416, + "nauc_recall_at_1000_max": 0.15815, + "nauc_recall_at_1000_std": 0.3002, + "nauc_recall_at_1000_diff1": 0.138179, + "nauc_precision_at_1_max": 0.305281, + "nauc_precision_at_1_std": -0.102015, + "nauc_precision_at_1_diff1": 0.540248, + "nauc_precision_at_3_max": 0.067297, + "nauc_precision_at_3_std": 0.104885, + "nauc_precision_at_3_diff1": -0.201175, + "nauc_precision_at_5_max": 0.060916, + "nauc_precision_at_5_std": 0.23969, + "nauc_precision_at_5_diff1": -0.321663, + "nauc_precision_at_10_max": 0.088951, + "nauc_precision_at_10_std": 0.395698, + "nauc_precision_at_10_diff1": -0.362, + "nauc_precision_at_20_max": 0.012009, + "nauc_precision_at_20_std": 0.423477, + "nauc_precision_at_20_diff1": -0.39259, + "nauc_precision_at_100_max": -0.091374, + "nauc_precision_at_100_std": 0.353063, + "nauc_precision_at_100_diff1": -0.36801, + "nauc_precision_at_1000_max": -0.223941, + "nauc_precision_at_1000_std": 0.176995, + "nauc_precision_at_1000_diff1": -0.340396, + "nauc_mrr_at_1_max": 0.305281, + "nauc_mrr_at_1_std": -0.102015, + "nauc_mrr_at_1_diff1": 0.540248, + "nauc_mrr_at_3_max": 0.321578, + "nauc_mrr_at_3_std": -0.086383, + "nauc_mrr_at_3_diff1": 0.55832, + "nauc_mrr_at_5_max": 0.320576, + "nauc_mrr_at_5_std": -0.068602, + "nauc_mrr_at_5_diff1": 0.544431, + "nauc_mrr_at_10_max": 0.33179, + "nauc_mrr_at_10_std": -0.06714, + "nauc_mrr_at_10_diff1": 0.546822, + "nauc_mrr_at_20_max": 0.331668, + "nauc_mrr_at_20_std": -0.069994, + "nauc_mrr_at_20_diff1": 0.547687, + "nauc_mrr_at_100_max": 0.328969, + "nauc_mrr_at_100_std": -0.073475, + "nauc_mrr_at_100_diff1": 0.546546, + "nauc_mrr_at_1000_max": 0.328969, + "nauc_mrr_at_1000_std": -0.073475, + "nauc_mrr_at_1000_diff1": 0.546546, + "hit_rate_at_1": 0.83511, + "hit_rate_at_3": 0.94149, + "hit_rate_at_5": 0.96277, + "hit_rate_at_10": 0.97872, + "hit_rate_at_20": 0.98404, + "hit_rate_at_100": 1.0, + "hit_rate_at_1000": 1.0, + "main_score": 0.57179, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ], + "real_chat_test": [ + { + "ndcg_at_1": 0.77586, + "ndcg_at_3": 0.71112, + "ndcg_at_5": 0.67687, + "ndcg_at_10": 0.61439, + "ndcg_at_20": 0.58107, + "ndcg_at_100": 0.53542, + "ndcg_at_1000": 0.6156, + "map_at_1": 0.06787, + "map_at_3": 0.11769, + "map_at_5": 0.14212, + "map_at_10": 0.16769, + "map_at_20": 0.19446, + "map_at_100": 0.25186, + "map_at_1000": 0.31441, + "recall_at_1": 0.06787, + "recall_at_3": 0.124, + "recall_at_5": 0.15975, + "recall_at_10": 0.20188, + "recall_at_20": 0.26602, + "recall_at_100": 0.43461, + "recall_at_1000": 0.75683, + "accuracy": 0.06787, + "precision_at_1": 0.77586, + "precision_at_3": 0.67672, + "precision_at_5": 0.61638, + "precision_at_10": 0.50431, + "precision_at_20": 0.42112, + "precision_at_100": 0.25177, + "precision_at_1000": 0.08669, + "mrr_at_1": 0.775862, + "mrr_at_3": 0.853448, + "mrr_at_5": 0.861422, + "mrr_at_10": 0.863398, + "mrr_at_20": 0.86373, + "mrr_at_100": 0.86373, + "mrr_at_1000": 0.863745, + "nauc_ndcg_at_1_max": 0.525907, + "nauc_ndcg_at_1_std": 0.269591, + "nauc_ndcg_at_1_diff1": 0.465375, + "nauc_ndcg_at_3_max": 0.287068, + "nauc_ndcg_at_3_std": 0.18443, + "nauc_ndcg_at_3_diff1": 0.121759, + "nauc_ndcg_at_5_max": 0.260473, + "nauc_ndcg_at_5_std": 0.159512, + "nauc_ndcg_at_5_diff1": 0.034001, + "nauc_ndcg_at_10_max": 0.340385, + "nauc_ndcg_at_10_std": 0.232896, + "nauc_ndcg_at_10_diff1": 0.032313, + "nauc_ndcg_at_20_max": 0.369467, + "nauc_ndcg_at_20_std": 0.241719, + "nauc_ndcg_at_20_diff1": 0.088464, + "nauc_ndcg_at_100_max": 0.350104, + "nauc_ndcg_at_100_std": 0.214528, + "nauc_ndcg_at_100_diff1": 0.126588, + "nauc_ndcg_at_1000_max": 0.311472, + "nauc_ndcg_at_1000_std": 0.318817, + "nauc_ndcg_at_1000_diff1": 0.154361, + "nauc_map_at_1_max": 0.413726, + "nauc_map_at_1_std": -0.081924, + "nauc_map_at_1_diff1": 0.457145, + "nauc_map_at_3_max": 0.289002, + "nauc_map_at_3_std": -0.094346, + "nauc_map_at_3_diff1": 0.332659, + "nauc_map_at_5_max": 0.284324, + "nauc_map_at_5_std": -0.080136, + "nauc_map_at_5_diff1": 0.325356, + "nauc_map_at_10_max": 0.294729, + "nauc_map_at_10_std": -0.056067, + "nauc_map_at_10_diff1": 0.296709, + "nauc_map_at_20_max": 0.299175, + "nauc_map_at_20_std": -0.025389, + "nauc_map_at_20_diff1": 0.300428, + "nauc_map_at_100_max": 0.315038, + "nauc_map_at_100_std": 0.084636, + "nauc_map_at_100_diff1": 0.265898, + "nauc_map_at_1000_max": 0.34538, + "nauc_map_at_1000_std": 0.182092, + "nauc_map_at_1000_diff1": 0.20962, + "nauc_recall_at_1_max": 0.413726, + "nauc_recall_at_1_std": -0.081924, + "nauc_recall_at_1_diff1": 0.457145, + "nauc_recall_at_3_max": 0.266582, + "nauc_recall_at_3_std": -0.106238, + "nauc_recall_at_3_diff1": 0.309085, + "nauc_recall_at_5_max": 0.256339, + "nauc_recall_at_5_std": -0.087993, + "nauc_recall_at_5_diff1": 0.283363, + "nauc_recall_at_10_max": 0.271208, + "nauc_recall_at_10_std": -0.046367, + "nauc_recall_at_10_diff1": 0.252427, + "nauc_recall_at_20_max": 0.27047, + "nauc_recall_at_20_std": -0.013823, + "nauc_recall_at_20_diff1": 0.271644, + "nauc_recall_at_100_max": 0.230345, + "nauc_recall_at_100_std": 0.114984, + "nauc_recall_at_100_diff1": 0.214268, + "nauc_recall_at_1000_max": 0.192038, + "nauc_recall_at_1000_std": 0.297009, + "nauc_recall_at_1000_diff1": 0.136686, + "nauc_precision_at_1_max": 0.525907, + "nauc_precision_at_1_std": 0.269591, + "nauc_precision_at_1_diff1": 0.465375, + "nauc_precision_at_3_max": 0.133824, + "nauc_precision_at_3_std": 0.183583, + "nauc_precision_at_3_diff1": -0.032448, + "nauc_precision_at_5_max": 0.088444, + "nauc_precision_at_5_std": 0.169706, + "nauc_precision_at_5_diff1": -0.139781, + "nauc_precision_at_10_max": 0.139893, + "nauc_precision_at_10_std": 0.269329, + "nauc_precision_at_10_diff1": -0.184645, + "nauc_precision_at_20_max": 0.097913, + "nauc_precision_at_20_std": 0.284943, + "nauc_precision_at_20_diff1": -0.185776, + "nauc_precision_at_100_max": 0.018281, + "nauc_precision_at_100_std": 0.242292, + "nauc_precision_at_100_diff1": -0.243258, + "nauc_precision_at_1000_max": -0.002772, + "nauc_precision_at_1000_std": 0.085361, + "nauc_precision_at_1000_diff1": -0.24366, + "nauc_mrr_at_1_max": 0.525907, + "nauc_mrr_at_1_std": 0.269591, + "nauc_mrr_at_1_diff1": 0.465375, + "nauc_mrr_at_3_max": 0.459522, + "nauc_mrr_at_3_std": 0.188, + "nauc_mrr_at_3_diff1": 0.425444, + "nauc_mrr_at_5_max": 0.462256, + "nauc_mrr_at_5_std": 0.179789, + "nauc_mrr_at_5_diff1": 0.422711, + "nauc_mrr_at_10_max": 0.482056, + "nauc_mrr_at_10_std": 0.207352, + "nauc_mrr_at_10_diff1": 0.431732, + "nauc_mrr_at_20_max": 0.480695, + "nauc_mrr_at_20_std": 0.207575, + "nauc_mrr_at_20_diff1": 0.431926, + "nauc_mrr_at_100_max": 0.480695, + "nauc_mrr_at_100_std": 0.207575, + "nauc_mrr_at_100_diff1": 0.431926, + "nauc_mrr_at_1000_max": 0.48071, + "nauc_mrr_at_1000_std": 0.207556, + "nauc_mrr_at_1000_diff1": 0.431889, + "hit_rate_at_1": 0.77586, + "hit_rate_at_3": 0.94397, + "hit_rate_at_5": 0.97845, + "hit_rate_at_10": 0.99138, + "hit_rate_at_20": 0.99569, + "hit_rate_at_100": 0.99569, + "hit_rate_at_1000": 1.0, + "main_score": 0.61439, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 890.31471824646, + "kg_co2_emissions": null, + "date": 1777988990.984288 +} \ No newline at end of file diff --git a/results/jinaai__jina-embeddings-v5-text-small/dd76d535f5447ca3897a9c893fb1e612ead98192/GlobalNewsRetrieval.json b/results/jinaai__jina-embeddings-v5-text-small/dd76d535f5447ca3897a9c893fb1e612ead98192/GlobalNewsRetrieval.json new file mode 100644 index 0000000000..b906d180d0 --- /dev/null +++ b/results/jinaai__jina-embeddings-v5-text-small/dd76d535f5447ca3897a9c893fb1e612ead98192/GlobalNewsRetrieval.json @@ -0,0 +1,1562 @@ +{ + "dataset_revision": "234dcd32922d32c78e08bfe506a2b90a5401cf7c", + "task_name": "GlobalNewsRetrieval", + "mteb_version": "2.12.37", + "scores": { + "test": [ + { + "ndcg_at_1": 0.67293, + "ndcg_at_3": 0.75228, + "ndcg_at_5": 0.77921, + "ndcg_at_10": 0.79889, + "ndcg_at_20": 0.80715, + "ndcg_at_100": 0.8149, + "ndcg_at_1000": 0.81569, + "map_at_1": 0.63952, + "map_at_3": 0.72435, + "map_at_5": 0.7418, + "map_at_10": 0.75101, + "map_at_20": 0.75366, + "map_at_100": 0.75502, + "map_at_1000": 0.75507, + "recall_at_1": 0.63952, + "recall_at_3": 0.8089, + "recall_at_5": 0.86967, + "recall_at_10": 0.9269, + "recall_at_20": 0.95718, + "recall_at_100": 0.99457, + "recall_at_1000": 1.0, + "accuracy": 0.63952, + "precision_at_1": 0.67293, + "precision_at_3": 0.29282, + "precision_at_5": 0.19198, + "precision_at_10": 0.10338, + "precision_at_20": 0.0537, + "precision_at_100": 0.01124, + "precision_at_1000": 0.00113, + "mrr_at_1": 0.672932, + "mrr_at_3": 0.748747, + "mrr_at_5": 0.760589, + "mrr_at_10": 0.767339, + "mrr_at_20": 0.769284, + "mrr_at_100": 0.770217, + "mrr_at_1000": 0.770238, + "nauc_ndcg_at_1_max": 0.465006, + "nauc_ndcg_at_1_std": -0.305125, + "nauc_ndcg_at_1_diff1": 0.788876, + "nauc_ndcg_at_3_max": 0.455528, + "nauc_ndcg_at_3_std": -0.352638, + "nauc_ndcg_at_3_diff1": 0.748742, + "nauc_ndcg_at_5_max": 0.471564, + "nauc_ndcg_at_5_std": -0.346317, + "nauc_ndcg_at_5_diff1": 0.751581, + "nauc_ndcg_at_10_max": 0.47906, + "nauc_ndcg_at_10_std": -0.321261, + "nauc_ndcg_at_10_diff1": 0.75931, + "nauc_ndcg_at_20_max": 0.471932, + "nauc_ndcg_at_20_std": -0.329955, + "nauc_ndcg_at_20_diff1": 0.758031, + "nauc_ndcg_at_100_max": 0.469003, + "nauc_ndcg_at_100_std": -0.331688, + "nauc_ndcg_at_100_diff1": 0.759498, + "nauc_ndcg_at_1000_max": 0.470001, + "nauc_ndcg_at_1000_std": -0.3287, + "nauc_ndcg_at_1000_diff1": 0.759773, + "nauc_map_at_1_max": 0.39053, + "nauc_map_at_1_std": -0.322333, + "nauc_map_at_1_diff1": 0.776988, + "nauc_map_at_3_max": 0.432744, + "nauc_map_at_3_std": -0.353773, + "nauc_map_at_3_diff1": 0.758655, + "nauc_map_at_5_max": 0.447581, + "nauc_map_at_5_std": -0.348342, + "nauc_map_at_5_diff1": 0.759591, + "nauc_map_at_10_max": 0.451271, + "nauc_map_at_10_std": -0.340223, + "nauc_map_at_10_diff1": 0.762027, + "nauc_map_at_20_max": 0.449446, + "nauc_map_at_20_std": -0.342531, + "nauc_map_at_20_diff1": 0.761581, + "nauc_map_at_100_max": 0.449324, + "nauc_map_at_100_std": -0.342139, + "nauc_map_at_100_diff1": 0.761784, + "nauc_map_at_1000_max": 0.449364, + "nauc_map_at_1000_std": -0.341984, + "nauc_map_at_1000_diff1": 0.761805, + "nauc_recall_at_1_max": 0.39053, + "nauc_recall_at_1_std": -0.322333, + "nauc_recall_at_1_diff1": 0.776988, + "nauc_recall_at_3_max": 0.453107, + "nauc_recall_at_3_std": -0.375256, + "nauc_recall_at_3_diff1": 0.697565, + "nauc_recall_at_5_max": 0.491042, + "nauc_recall_at_5_std": -0.379035, + "nauc_recall_at_5_diff1": 0.696347, + "nauc_recall_at_10_max": 0.553546, + "nauc_recall_at_10_std": -0.217679, + "nauc_recall_at_10_diff1": 0.719196, + "nauc_recall_at_20_max": 0.512161, + "nauc_recall_at_20_std": -0.297964, + "nauc_recall_at_20_diff1": 0.698044, + "nauc_recall_at_100_max": 0.241944, + "nauc_recall_at_100_std": -0.857052, + "nauc_recall_at_100_diff1": 0.671114, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.465006, + "nauc_precision_at_1_std": -0.305125, + "nauc_precision_at_1_diff1": 0.788876, + "nauc_precision_at_3_max": 0.479575, + "nauc_precision_at_3_std": -0.229039, + "nauc_precision_at_3_diff1": 0.431909, + "nauc_precision_at_5_max": 0.420052, + "nauc_precision_at_5_std": -0.084915, + "nauc_precision_at_5_diff1": 0.205952, + "nauc_precision_at_10_max": 0.324372, + "nauc_precision_at_10_std": 0.068253, + "nauc_precision_at_10_diff1": 0.009898, + "nauc_precision_at_20_max": 0.236803, + "nauc_precision_at_20_std": 0.103191, + "nauc_precision_at_20_diff1": -0.119645, + "nauc_precision_at_100_max": 0.132317, + "nauc_precision_at_100_std": 0.191541, + "nauc_precision_at_100_diff1": -0.267751, + "nauc_precision_at_1000_max": 0.123, + "nauc_precision_at_1000_std": 0.221705, + "nauc_precision_at_1000_diff1": -0.287155, + "nauc_mrr_at_1_max": 0.465006, + "nauc_mrr_at_1_std": -0.305125, + "nauc_mrr_at_1_diff1": 0.788876, + "nauc_mrr_at_3_max": 0.494645, + "nauc_mrr_at_3_std": -0.321758, + "nauc_mrr_at_3_diff1": 0.757459, + "nauc_mrr_at_5_max": 0.492181, + "nauc_mrr_at_5_std": -0.318938, + "nauc_mrr_at_5_diff1": 0.759875, + "nauc_mrr_at_10_max": 0.492874, + "nauc_mrr_at_10_std": -0.30859, + "nauc_mrr_at_10_diff1": 0.763694, + "nauc_mrr_at_20_max": 0.49097, + "nauc_mrr_at_20_std": -0.309915, + "nauc_mrr_at_20_diff1": 0.763721, + "nauc_mrr_at_100_max": 0.490315, + "nauc_mrr_at_100_std": -0.310587, + "nauc_mrr_at_100_diff1": 0.763723, + "nauc_mrr_at_1000_max": 0.49035, + "nauc_mrr_at_1000_std": -0.310517, + "nauc_mrr_at_1000_diff1": 0.76373, + "hit_rate_at_1": 0.67293, + "hit_rate_at_3": 0.83835, + "hit_rate_at_5": 0.88972, + "hit_rate_at_10": 0.93985, + "hit_rate_at_20": 0.96742, + "hit_rate_at_100": 0.99624, + "hit_rate_at_1000": 1.0, + "main_score": 0.79889, + "hf_subset": "ar", + "languages": [ + "ara-Arab" + ] + }, + { + "ndcg_at_1": 0.81183, + "ndcg_at_3": 0.85396, + "ndcg_at_5": 0.86742, + "ndcg_at_10": 0.87608, + "ndcg_at_20": 0.88239, + "ndcg_at_100": 0.88988, + "ndcg_at_1000": 0.89041, + "map_at_1": 0.75242, + "map_at_3": 0.83498, + "map_at_5": 0.84414, + "map_at_10": 0.84876, + "map_at_20": 0.85101, + "map_at_100": 0.85229, + "map_at_1000": 0.85233, + "recall_at_1": 0.75242, + "recall_at_3": 0.88504, + "recall_at_5": 0.91461, + "recall_at_10": 0.93826, + "recall_at_20": 0.96004, + "recall_at_100": 0.99695, + "recall_at_1000": 1.0, + "accuracy": 0.75242, + "precision_at_1": 0.81183, + "precision_at_3": 0.33656, + "precision_at_5": 0.21075, + "precision_at_10": 0.10903, + "precision_at_20": 0.05629, + "precision_at_100": 0.01177, + "precision_at_1000": 0.00118, + "mrr_at_1": 0.811828, + "mrr_at_3": 0.858244, + "mrr_at_5": 0.863889, + "mrr_at_10": 0.866444, + "mrr_at_20": 0.867704, + "mrr_at_100": 0.868441, + "mrr_at_1000": 0.868448, + "nauc_ndcg_at_1_max": 0.489136, + "nauc_ndcg_at_1_std": -0.597764, + "nauc_ndcg_at_1_diff1": 0.843417, + "nauc_ndcg_at_3_max": 0.510593, + "nauc_ndcg_at_3_std": -0.645033, + "nauc_ndcg_at_3_diff1": 0.818977, + "nauc_ndcg_at_5_max": 0.526159, + "nauc_ndcg_at_5_std": -0.660676, + "nauc_ndcg_at_5_diff1": 0.81877, + "nauc_ndcg_at_10_max": 0.530951, + "nauc_ndcg_at_10_std": -0.651705, + "nauc_ndcg_at_10_diff1": 0.825125, + "nauc_ndcg_at_20_max": 0.525618, + "nauc_ndcg_at_20_std": -0.648215, + "nauc_ndcg_at_20_diff1": 0.825617, + "nauc_ndcg_at_100_max": 0.517447, + "nauc_ndcg_at_100_std": -0.634325, + "nauc_ndcg_at_100_diff1": 0.825805, + "nauc_ndcg_at_1000_max": 0.520593, + "nauc_ndcg_at_1000_std": -0.634167, + "nauc_ndcg_at_1000_diff1": 0.825625, + "nauc_map_at_1_max": 0.365998, + "nauc_map_at_1_std": -0.522778, + "nauc_map_at_1_diff1": 0.815662, + "nauc_map_at_3_max": 0.490197, + "nauc_map_at_3_std": -0.620268, + "nauc_map_at_3_diff1": 0.816648, + "nauc_map_at_5_max": 0.504853, + "nauc_map_at_5_std": -0.628231, + "nauc_map_at_5_diff1": 0.816238, + "nauc_map_at_10_max": 0.509874, + "nauc_map_at_10_std": -0.624139, + "nauc_map_at_10_diff1": 0.819908, + "nauc_map_at_20_max": 0.507958, + "nauc_map_at_20_std": -0.623958, + "nauc_map_at_20_diff1": 0.820258, + "nauc_map_at_100_max": 0.507095, + "nauc_map_at_100_std": -0.621964, + "nauc_map_at_100_diff1": 0.820251, + "nauc_map_at_1000_max": 0.507339, + "nauc_map_at_1000_std": -0.621944, + "nauc_map_at_1000_diff1": 0.820245, + "nauc_recall_at_1_max": 0.365998, + "nauc_recall_at_1_std": -0.522778, + "nauc_recall_at_1_diff1": 0.815662, + "nauc_recall_at_3_max": 0.53045, + "nauc_recall_at_3_std": -0.694923, + "nauc_recall_at_3_diff1": 0.793185, + "nauc_recall_at_5_max": 0.56554, + "nauc_recall_at_5_std": -0.771218, + "nauc_recall_at_5_diff1": 0.77788, + "nauc_recall_at_10_max": 0.596271, + "nauc_recall_at_10_std": -0.770389, + "nauc_recall_at_10_diff1": 0.801236, + "nauc_recall_at_20_max": 0.596865, + "nauc_recall_at_20_std": -0.800896, + "nauc_recall_at_20_diff1": 0.810675, + "nauc_recall_at_100_max": -0.076591, + "nauc_recall_at_100_std": -0.585242, + "nauc_recall_at_100_diff1": 0.823008, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.489136, + "nauc_precision_at_1_std": -0.597764, + "nauc_precision_at_1_diff1": 0.843417, + "nauc_precision_at_3_max": 0.265644, + "nauc_precision_at_3_std": -0.212921, + "nauc_precision_at_3_diff1": 0.073366, + "nauc_precision_at_5_max": 0.224019, + "nauc_precision_at_5_std": -0.126389, + "nauc_precision_at_5_diff1": -0.049083, + "nauc_precision_at_10_max": 0.169455, + "nauc_precision_at_10_std": -0.024742, + "nauc_precision_at_10_diff1": -0.140799, + "nauc_precision_at_20_max": 0.087347, + "nauc_precision_at_20_std": 0.054087, + "nauc_precision_at_20_diff1": -0.235002, + "nauc_precision_at_100_max": -0.023151, + "nauc_precision_at_100_std": 0.209939, + "nauc_precision_at_100_diff1": -0.379701, + "nauc_precision_at_1000_max": -0.017092, + "nauc_precision_at_1000_std": 0.220479, + "nauc_precision_at_1000_diff1": -0.386837, + "nauc_mrr_at_1_max": 0.489136, + "nauc_mrr_at_1_std": -0.597764, + "nauc_mrr_at_1_diff1": 0.843417, + "nauc_mrr_at_3_max": 0.528713, + "nauc_mrr_at_3_std": -0.645539, + "nauc_mrr_at_3_diff1": 0.837792, + "nauc_mrr_at_5_max": 0.529261, + "nauc_mrr_at_5_std": -0.650201, + "nauc_mrr_at_5_diff1": 0.838106, + "nauc_mrr_at_10_max": 0.526915, + "nauc_mrr_at_10_std": -0.647085, + "nauc_mrr_at_10_diff1": 0.839581, + "nauc_mrr_at_20_max": 0.526037, + "nauc_mrr_at_20_std": -0.644868, + "nauc_mrr_at_20_diff1": 0.839558, + "nauc_mrr_at_100_max": 0.525262, + "nauc_mrr_at_100_std": -0.643542, + "nauc_mrr_at_100_diff1": 0.8394, + "nauc_mrr_at_1000_max": 0.52526, + "nauc_mrr_at_1000_std": -0.643586, + "nauc_mrr_at_1000_diff1": 0.839391, + "hit_rate_at_1": 0.81183, + "hit_rate_at_3": 0.90968, + "hit_rate_at_5": 0.93441, + "hit_rate_at_10": 0.95376, + "hit_rate_at_20": 0.97097, + "hit_rate_at_100": 0.99892, + "hit_rate_at_1000": 1.0, + "main_score": 0.87608, + "hf_subset": "zh", + "languages": [ + "zho-Hans" + ] + }, + { + "ndcg_at_1": 0.76876, + "ndcg_at_3": 0.82029, + "ndcg_at_5": 0.84036, + "ndcg_at_10": 0.85205, + "ndcg_at_20": 0.86008, + "ndcg_at_100": 0.86587, + "ndcg_at_1000": 0.86633, + "map_at_1": 0.74084, + "map_at_3": 0.80173, + "map_at_5": 0.81385, + "map_at_10": 0.81975, + "map_at_20": 0.8221, + "map_at_100": 0.82312, + "map_at_1000": 0.82315, + "recall_at_1": 0.74084, + "recall_at_3": 0.85834, + "recall_at_5": 0.90425, + "recall_at_10": 0.93722, + "recall_at_20": 0.96848, + "recall_at_100": 0.99692, + "recall_at_1000": 1.0, + "accuracy": 0.74084, + "precision_at_1": 0.76876, + "precision_at_3": 0.30524, + "precision_at_5": 0.19424, + "precision_at_10": 0.10164, + "precision_at_20": 0.05267, + "precision_at_100": 0.01089, + "precision_at_1000": 0.00109, + "mrr_at_1": 0.768756, + "mrr_at_3": 0.817575, + "mrr_at_5": 0.828058, + "mrr_at_10": 0.8315, + "mrr_at_20": 0.833351, + "mrr_at_100": 0.834028, + "mrr_at_1000": 0.834044, + "nauc_ndcg_at_1_max": 0.500369, + "nauc_ndcg_at_1_std": -0.459028, + "nauc_ndcg_at_1_diff1": 0.802176, + "nauc_ndcg_at_3_max": 0.503411, + "nauc_ndcg_at_3_std": -0.533551, + "nauc_ndcg_at_3_diff1": 0.783765, + "nauc_ndcg_at_5_max": 0.506756, + "nauc_ndcg_at_5_std": -0.529486, + "nauc_ndcg_at_5_diff1": 0.784276, + "nauc_ndcg_at_10_max": 0.508121, + "nauc_ndcg_at_10_std": -0.526546, + "nauc_ndcg_at_10_diff1": 0.784417, + "nauc_ndcg_at_20_max": 0.509949, + "nauc_ndcg_at_20_std": -0.504519, + "nauc_ndcg_at_20_diff1": 0.784268, + "nauc_ndcg_at_100_max": 0.505773, + "nauc_ndcg_at_100_std": -0.505692, + "nauc_ndcg_at_100_diff1": 0.785473, + "nauc_ndcg_at_1000_max": 0.505539, + "nauc_ndcg_at_1000_std": -0.50524, + "nauc_ndcg_at_1000_diff1": 0.786752, + "nauc_map_at_1_max": 0.431315, + "nauc_map_at_1_std": -0.455897, + "nauc_map_at_1_diff1": 0.79178, + "nauc_map_at_3_max": 0.487824, + "nauc_map_at_3_std": -0.521564, + "nauc_map_at_3_diff1": 0.786278, + "nauc_map_at_5_max": 0.49151, + "nauc_map_at_5_std": -0.518444, + "nauc_map_at_5_diff1": 0.787781, + "nauc_map_at_10_max": 0.492665, + "nauc_map_at_10_std": -0.517203, + "nauc_map_at_10_diff1": 0.787652, + "nauc_map_at_20_max": 0.493573, + "nauc_map_at_20_std": -0.510745, + "nauc_map_at_20_diff1": 0.787556, + "nauc_map_at_100_max": 0.493014, + "nauc_map_at_100_std": -0.510229, + "nauc_map_at_100_diff1": 0.787577, + "nauc_map_at_1000_max": 0.493005, + "nauc_map_at_1000_std": -0.510194, + "nauc_map_at_1000_diff1": 0.787637, + "nauc_recall_at_1_max": 0.431315, + "nauc_recall_at_1_std": -0.455897, + "nauc_recall_at_1_diff1": 0.79178, + "nauc_recall_at_3_max": 0.517261, + "nauc_recall_at_3_std": -0.598219, + "nauc_recall_at_3_diff1": 0.75948, + "nauc_recall_at_5_max": 0.525102, + "nauc_recall_at_5_std": -0.61468, + "nauc_recall_at_5_diff1": 0.745092, + "nauc_recall_at_10_max": 0.537414, + "nauc_recall_at_10_std": -0.639437, + "nauc_recall_at_10_diff1": 0.732226, + "nauc_recall_at_20_max": 0.592794, + "nauc_recall_at_20_std": -0.432525, + "nauc_recall_at_20_diff1": 0.6966, + "nauc_recall_at_100_max": 0.561648, + "nauc_recall_at_100_std": -0.583165, + "nauc_recall_at_100_diff1": 0.410974, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.500369, + "nauc_precision_at_1_std": -0.459028, + "nauc_precision_at_1_diff1": 0.802176, + "nauc_precision_at_3_max": 0.45236, + "nauc_precision_at_3_std": -0.320343, + "nauc_precision_at_3_diff1": 0.390688, + "nauc_precision_at_5_max": 0.384018, + "nauc_precision_at_5_std": -0.180732, + "nauc_precision_at_5_diff1": 0.207336, + "nauc_precision_at_10_max": 0.247373, + "nauc_precision_at_10_std": -0.02644, + "nauc_precision_at_10_diff1": 0.002437, + "nauc_precision_at_20_max": 0.168706, + "nauc_precision_at_20_std": 0.178, + "nauc_precision_at_20_diff1": -0.177885, + "nauc_precision_at_100_max": 0.040891, + "nauc_precision_at_100_std": 0.282149, + "nauc_precision_at_100_diff1": -0.333385, + "nauc_precision_at_1000_max": 0.02835, + "nauc_precision_at_1000_std": 0.292877, + "nauc_precision_at_1000_diff1": -0.339572, + "nauc_mrr_at_1_max": 0.500369, + "nauc_mrr_at_1_std": -0.459028, + "nauc_mrr_at_1_diff1": 0.802176, + "nauc_mrr_at_3_max": 0.520249, + "nauc_mrr_at_3_std": -0.502448, + "nauc_mrr_at_3_diff1": 0.792221, + "nauc_mrr_at_5_max": 0.518252, + "nauc_mrr_at_5_std": -0.497871, + "nauc_mrr_at_5_diff1": 0.791689, + "nauc_mrr_at_10_max": 0.518441, + "nauc_mrr_at_10_std": -0.496828, + "nauc_mrr_at_10_diff1": 0.792332, + "nauc_mrr_at_20_max": 0.517895, + "nauc_mrr_at_20_std": -0.493349, + "nauc_mrr_at_20_diff1": 0.792404, + "nauc_mrr_at_100_max": 0.517542, + "nauc_mrr_at_100_std": -0.49339, + "nauc_mrr_at_100_diff1": 0.79251, + "nauc_mrr_at_1000_max": 0.517521, + "nauc_mrr_at_1000_std": -0.493343, + "nauc_mrr_at_1000_diff1": 0.79253, + "hit_rate_at_1": 0.76876, + "hit_rate_at_3": 0.8777, + "hit_rate_at_5": 0.92189, + "hit_rate_at_10": 0.94758, + "hit_rate_at_20": 0.97636, + "hit_rate_at_100": 0.99794, + "hit_rate_at_1000": 1.0, + "main_score": 0.85205, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.74259, + "ndcg_at_3": 0.8261, + "ndcg_at_5": 0.84199, + "ndcg_at_10": 0.85526, + "ndcg_at_20": 0.86058, + "ndcg_at_100": 0.86414, + "ndcg_at_1000": 0.86482, + "map_at_1": 0.71995, + "map_at_3": 0.80215, + "map_at_5": 0.8118, + "map_at_10": 0.81811, + "map_at_20": 0.81987, + "map_at_100": 0.8205, + "map_at_1000": 0.82054, + "recall_at_1": 0.71995, + "recall_at_3": 0.88304, + "recall_at_5": 0.91965, + "recall_at_10": 0.95863, + "recall_at_20": 0.97804, + "recall_at_100": 0.9954, + "recall_at_1000": 1.0, + "accuracy": 0.71995, + "precision_at_1": 0.74259, + "precision_at_3": 0.31086, + "precision_at_5": 0.19489, + "precision_at_10": 0.10215, + "precision_at_20": 0.05235, + "precision_at_100": 0.01067, + "precision_at_1000": 0.00107, + "mrr_at_1": 0.742594, + "mrr_at_3": 0.813756, + "mrr_at_5": 0.821978, + "mrr_at_10": 0.826512, + "mrr_at_20": 0.827614, + "mrr_at_100": 0.828115, + "mrr_at_1000": 0.828148, + "nauc_ndcg_at_1_max": 0.460314, + "nauc_ndcg_at_1_std": -0.322016, + "nauc_ndcg_at_1_diff1": 0.813033, + "nauc_ndcg_at_3_max": 0.492124, + "nauc_ndcg_at_3_std": -0.407509, + "nauc_ndcg_at_3_diff1": 0.77759, + "nauc_ndcg_at_5_max": 0.498198, + "nauc_ndcg_at_5_std": -0.392062, + "nauc_ndcg_at_5_diff1": 0.783234, + "nauc_ndcg_at_10_max": 0.496052, + "nauc_ndcg_at_10_std": -0.399292, + "nauc_ndcg_at_10_diff1": 0.786212, + "nauc_ndcg_at_20_max": 0.495094, + "nauc_ndcg_at_20_std": -0.375679, + "nauc_ndcg_at_20_diff1": 0.78657, + "nauc_ndcg_at_100_max": 0.49135, + "nauc_ndcg_at_100_std": -0.36597, + "nauc_ndcg_at_100_diff1": 0.787816, + "nauc_ndcg_at_1000_max": 0.489731, + "nauc_ndcg_at_1000_std": -0.36923, + "nauc_ndcg_at_1000_diff1": 0.78851, + "nauc_map_at_1_max": 0.41728, + "nauc_map_at_1_std": -0.322172, + "nauc_map_at_1_diff1": 0.798111, + "nauc_map_at_3_max": 0.478184, + "nauc_map_at_3_std": -0.383341, + "nauc_map_at_3_diff1": 0.783415, + "nauc_map_at_5_max": 0.481583, + "nauc_map_at_5_std": -0.375003, + "nauc_map_at_5_diff1": 0.78669, + "nauc_map_at_10_max": 0.482181, + "nauc_map_at_10_std": -0.376727, + "nauc_map_at_10_diff1": 0.78835, + "nauc_map_at_20_max": 0.482182, + "nauc_map_at_20_std": -0.369997, + "nauc_map_at_20_diff1": 0.788443, + "nauc_map_at_100_max": 0.481517, + "nauc_map_at_100_std": -0.368717, + "nauc_map_at_100_diff1": 0.788452, + "nauc_map_at_1000_max": 0.481452, + "nauc_map_at_1000_std": -0.36882, + "nauc_map_at_1000_diff1": 0.788477, + "nauc_recall_at_1_max": 0.41728, + "nauc_recall_at_1_std": -0.322172, + "nauc_recall_at_1_diff1": 0.798111, + "nauc_recall_at_3_max": 0.530447, + "nauc_recall_at_3_std": -0.521097, + "nauc_recall_at_3_diff1": 0.73557, + "nauc_recall_at_5_max": 0.561756, + "nauc_recall_at_5_std": -0.504088, + "nauc_recall_at_5_diff1": 0.733655, + "nauc_recall_at_10_max": 0.592986, + "nauc_recall_at_10_std": -0.696583, + "nauc_recall_at_10_diff1": 0.72739, + "nauc_recall_at_20_max": 0.654219, + "nauc_recall_at_20_std": -0.493362, + "nauc_recall_at_20_diff1": 0.706455, + "nauc_recall_at_100_max": 0.80049, + "nauc_recall_at_100_std": 0.282144, + "nauc_recall_at_100_diff1": 0.65287, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.460314, + "nauc_precision_at_1_std": -0.322016, + "nauc_precision_at_1_diff1": 0.813033, + "nauc_precision_at_3_max": 0.39619, + "nauc_precision_at_3_std": -0.291185, + "nauc_precision_at_3_diff1": 0.355288, + "nauc_precision_at_5_max": 0.341782, + "nauc_precision_at_5_std": -0.178301, + "nauc_precision_at_5_diff1": 0.20881, + "nauc_precision_at_10_max": 0.199762, + "nauc_precision_at_10_std": -0.089326, + "nauc_precision_at_10_diff1": -0.025117, + "nauc_precision_at_20_max": 0.107645, + "nauc_precision_at_20_std": 0.102411, + "nauc_precision_at_20_diff1": -0.166071, + "nauc_precision_at_100_max": 0.012151, + "nauc_precision_at_100_std": 0.213094, + "nauc_precision_at_100_diff1": -0.287809, + "nauc_precision_at_1000_max": -0.018258, + "nauc_precision_at_1000_std": 0.213755, + "nauc_precision_at_1000_diff1": -0.320403, + "nauc_mrr_at_1_max": 0.460314, + "nauc_mrr_at_1_std": -0.322016, + "nauc_mrr_at_1_diff1": 0.813033, + "nauc_mrr_at_3_max": 0.490338, + "nauc_mrr_at_3_std": -0.375276, + "nauc_mrr_at_3_diff1": 0.792764, + "nauc_mrr_at_5_max": 0.492669, + "nauc_mrr_at_5_std": -0.364432, + "nauc_mrr_at_5_diff1": 0.795973, + "nauc_mrr_at_10_max": 0.490497, + "nauc_mrr_at_10_std": -0.36595, + "nauc_mrr_at_10_diff1": 0.796817, + "nauc_mrr_at_20_max": 0.489838, + "nauc_mrr_at_20_std": -0.36217, + "nauc_mrr_at_20_diff1": 0.79676, + "nauc_mrr_at_100_max": 0.489617, + "nauc_mrr_at_100_std": -0.360563, + "nauc_mrr_at_100_diff1": 0.796823, + "nauc_mrr_at_1000_max": 0.489544, + "nauc_mrr_at_1000_std": -0.360752, + "nauc_mrr_at_1000_diff1": 0.796856, + "hit_rate_at_1": 0.74259, + "hit_rate_at_3": 0.89683, + "hit_rate_at_5": 0.93156, + "hit_rate_at_10": 0.96527, + "hit_rate_at_20": 0.98059, + "hit_rate_at_100": 0.99591, + "hit_rate_at_1000": 1.0, + "main_score": 0.85526, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.75318, + "ndcg_at_3": 0.81372, + "ndcg_at_5": 0.8272, + "ndcg_at_10": 0.83811, + "ndcg_at_20": 0.84806, + "ndcg_at_100": 0.85419, + "ndcg_at_1000": 0.85548, + "map_at_1": 0.73023, + "map_at_3": 0.79617, + "map_at_5": 0.80414, + "map_at_10": 0.809, + "map_at_20": 0.81202, + "map_at_100": 0.81286, + "map_at_1000": 0.81292, + "recall_at_1": 0.73023, + "recall_at_3": 0.85734, + "recall_at_5": 0.88833, + "recall_at_10": 0.92064, + "recall_at_20": 0.95825, + "recall_at_100": 0.99073, + "recall_at_1000": 1.0, + "accuracy": 0.73023, + "precision_at_1": 0.75318, + "precision_at_3": 0.30085, + "precision_at_5": 0.18792, + "precision_at_10": 0.09799, + "precision_at_20": 0.05111, + "precision_at_100": 0.01062, + "precision_at_1000": 0.00107, + "mrr_at_1": 0.753178, + "mrr_at_3": 0.803496, + "mrr_at_5": 0.810752, + "mrr_at_10": 0.814825, + "mrr_at_20": 0.817353, + "mrr_at_100": 0.817992, + "mrr_at_1000": 0.818046, + "nauc_ndcg_at_1_max": 0.508518, + "nauc_ndcg_at_1_std": -0.506637, + "nauc_ndcg_at_1_diff1": 0.819914, + "nauc_ndcg_at_3_max": 0.549116, + "nauc_ndcg_at_3_std": -0.609963, + "nauc_ndcg_at_3_diff1": 0.813901, + "nauc_ndcg_at_5_max": 0.544989, + "nauc_ndcg_at_5_std": -0.621019, + "nauc_ndcg_at_5_diff1": 0.806796, + "nauc_ndcg_at_10_max": 0.549218, + "nauc_ndcg_at_10_std": -0.615761, + "nauc_ndcg_at_10_diff1": 0.80006, + "nauc_ndcg_at_20_max": 0.545945, + "nauc_ndcg_at_20_std": -0.609314, + "nauc_ndcg_at_20_diff1": 0.809027, + "nauc_ndcg_at_100_max": 0.540436, + "nauc_ndcg_at_100_std": -0.590345, + "nauc_ndcg_at_100_diff1": 0.808022, + "nauc_ndcg_at_1000_max": 0.539849, + "nauc_ndcg_at_1000_std": -0.591804, + "nauc_ndcg_at_1000_diff1": 0.809402, + "nauc_map_at_1_max": 0.434404, + "nauc_map_at_1_std": -0.471471, + "nauc_map_at_1_diff1": 0.800464, + "nauc_map_at_3_max": 0.529036, + "nauc_map_at_3_std": -0.575784, + "nauc_map_at_3_diff1": 0.810676, + "nauc_map_at_5_max": 0.528493, + "nauc_map_at_5_std": -0.580321, + "nauc_map_at_5_diff1": 0.806867, + "nauc_map_at_10_max": 0.530027, + "nauc_map_at_10_std": -0.57846, + "nauc_map_at_10_diff1": 0.804134, + "nauc_map_at_20_max": 0.529044, + "nauc_map_at_20_std": -0.577223, + "nauc_map_at_20_diff1": 0.806736, + "nauc_map_at_100_max": 0.528486, + "nauc_map_at_100_std": -0.57487, + "nauc_map_at_100_diff1": 0.806651, + "nauc_map_at_1000_max": 0.528473, + "nauc_map_at_1000_std": -0.574939, + "nauc_map_at_1000_diff1": 0.806716, + "nauc_recall_at_1_max": 0.434404, + "nauc_recall_at_1_std": -0.471471, + "nauc_recall_at_1_diff1": 0.800464, + "nauc_recall_at_3_max": 0.593085, + "nauc_recall_at_3_std": -0.719791, + "nauc_recall_at_3_diff1": 0.807017, + "nauc_recall_at_5_max": 0.587019, + "nauc_recall_at_5_std": -0.785678, + "nauc_recall_at_5_diff1": 0.783157, + "nauc_recall_at_10_max": 0.627045, + "nauc_recall_at_10_std": -0.810931, + "nauc_recall_at_10_diff1": 0.735372, + "nauc_recall_at_20_max": 0.659065, + "nauc_recall_at_20_std": -0.888938, + "nauc_recall_at_20_diff1": 0.792255, + "nauc_recall_at_100_max": 0.610212, + "nauc_recall_at_100_std": -0.450397, + "nauc_recall_at_100_diff1": 0.668782, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.508518, + "nauc_precision_at_1_std": -0.506637, + "nauc_precision_at_1_diff1": 0.819914, + "nauc_precision_at_3_max": 0.505754, + "nauc_precision_at_3_std": -0.508323, + "nauc_precision_at_3_diff1": 0.465841, + "nauc_precision_at_5_max": 0.444705, + "nauc_precision_at_5_std": -0.4669, + "nauc_precision_at_5_diff1": 0.341688, + "nauc_precision_at_10_max": 0.373435, + "nauc_precision_at_10_std": -0.361805, + "nauc_precision_at_10_diff1": 0.16774, + "nauc_precision_at_20_max": 0.255932, + "nauc_precision_at_20_std": -0.221261, + "nauc_precision_at_20_diff1": -0.00442, + "nauc_precision_at_100_max": 0.067837, + "nauc_precision_at_100_std": 0.062294, + "nauc_precision_at_100_diff1": -0.258276, + "nauc_precision_at_1000_max": 0.027783, + "nauc_precision_at_1000_std": 0.099452, + "nauc_precision_at_1000_diff1": -0.318121, + "nauc_mrr_at_1_max": 0.508518, + "nauc_mrr_at_1_std": -0.506637, + "nauc_mrr_at_1_diff1": 0.819914, + "nauc_mrr_at_3_max": 0.553439, + "nauc_mrr_at_3_std": -0.590211, + "nauc_mrr_at_3_diff1": 0.823399, + "nauc_mrr_at_5_max": 0.547915, + "nauc_mrr_at_5_std": -0.594576, + "nauc_mrr_at_5_diff1": 0.820157, + "nauc_mrr_at_10_max": 0.548472, + "nauc_mrr_at_10_std": -0.591175, + "nauc_mrr_at_10_diff1": 0.818228, + "nauc_mrr_at_20_max": 0.547471, + "nauc_mrr_at_20_std": -0.588562, + "nauc_mrr_at_20_diff1": 0.820167, + "nauc_mrr_at_100_max": 0.547113, + "nauc_mrr_at_100_std": -0.58654, + "nauc_mrr_at_100_diff1": 0.820107, + "nauc_mrr_at_1000_max": 0.54704, + "nauc_mrr_at_1000_std": -0.586587, + "nauc_mrr_at_1000_diff1": 0.820121, + "hit_rate_at_1": 0.75318, + "hit_rate_at_3": 0.86547, + "hit_rate_at_5": 0.89619, + "hit_rate_at_10": 0.92797, + "hit_rate_at_20": 0.96292, + "hit_rate_at_100": 0.99153, + "hit_rate_at_1000": 1.0, + "main_score": 0.83811, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ] + }, + { + "ndcg_at_1": 0.63136, + "ndcg_at_3": 0.69639, + "ndcg_at_5": 0.71394, + "ndcg_at_10": 0.7299, + "ndcg_at_20": 0.7409, + "ndcg_at_100": 0.75822, + "ndcg_at_1000": 0.76333, + "map_at_1": 0.61059, + "map_at_3": 0.67564, + "map_at_5": 0.68571, + "map_at_10": 0.69258, + "map_at_20": 0.6958, + "map_at_100": 0.69842, + "map_at_1000": 0.69868, + "recall_at_1": 0.61059, + "recall_at_3": 0.74316, + "recall_at_5": 0.78522, + "recall_at_10": 0.83368, + "recall_at_20": 0.87522, + "recall_at_100": 0.96366, + "recall_at_1000": 1.0, + "accuracy": 0.61059, + "precision_at_1": 0.63136, + "precision_at_3": 0.25926, + "precision_at_5": 0.16469, + "precision_at_10": 0.08775, + "precision_at_20": 0.04631, + "precision_at_100": 0.01032, + "precision_at_1000": 0.00108, + "mrr_at_1": 0.63136, + "mrr_at_3": 0.689512, + "mrr_at_5": 0.69865, + "mrr_at_10": 0.704805, + "mrr_at_20": 0.707715, + "mrr_at_100": 0.710076, + "mrr_at_1000": 0.71026, + "nauc_ndcg_at_1_max": 0.524056, + "nauc_ndcg_at_1_std": -0.4011, + "nauc_ndcg_at_1_diff1": 0.783681, + "nauc_ndcg_at_3_max": 0.534528, + "nauc_ndcg_at_3_std": -0.450601, + "nauc_ndcg_at_3_diff1": 0.735208, + "nauc_ndcg_at_5_max": 0.540197, + "nauc_ndcg_at_5_std": -0.445084, + "nauc_ndcg_at_5_diff1": 0.728753, + "nauc_ndcg_at_10_max": 0.546705, + "nauc_ndcg_at_10_std": -0.444614, + "nauc_ndcg_at_10_diff1": 0.726625, + "nauc_ndcg_at_20_max": 0.542845, + "nauc_ndcg_at_20_std": -0.451374, + "nauc_ndcg_at_20_diff1": 0.729028, + "nauc_ndcg_at_100_max": 0.537503, + "nauc_ndcg_at_100_std": -0.452684, + "nauc_ndcg_at_100_diff1": 0.739424, + "nauc_ndcg_at_1000_max": 0.538205, + "nauc_ndcg_at_1000_std": -0.442583, + "nauc_ndcg_at_1000_diff1": 0.739321, + "nauc_map_at_1_max": 0.502413, + "nauc_map_at_1_std": -0.380661, + "nauc_map_at_1_diff1": 0.764392, + "nauc_map_at_3_max": 0.525729, + "nauc_map_at_3_std": -0.435597, + "nauc_map_at_3_diff1": 0.738428, + "nauc_map_at_5_max": 0.528766, + "nauc_map_at_5_std": -0.432662, + "nauc_map_at_5_diff1": 0.735613, + "nauc_map_at_10_max": 0.530742, + "nauc_map_at_10_std": -0.432753, + "nauc_map_at_10_diff1": 0.735113, + "nauc_map_at_20_max": 0.529792, + "nauc_map_at_20_std": -0.434222, + "nauc_map_at_20_diff1": 0.735693, + "nauc_map_at_100_max": 0.52929, + "nauc_map_at_100_std": -0.434, + "nauc_map_at_100_diff1": 0.737276, + "nauc_map_at_1000_max": 0.529363, + "nauc_map_at_1000_std": -0.433581, + "nauc_map_at_1000_diff1": 0.737292, + "nauc_recall_at_1_max": 0.502413, + "nauc_recall_at_1_std": -0.380661, + "nauc_recall_at_1_diff1": 0.764392, + "nauc_recall_at_3_max": 0.542391, + "nauc_recall_at_3_std": -0.490442, + "nauc_recall_at_3_diff1": 0.695875, + "nauc_recall_at_5_max": 0.56185, + "nauc_recall_at_5_std": -0.480022, + "nauc_recall_at_5_diff1": 0.671024, + "nauc_recall_at_10_max": 0.59522, + "nauc_recall_at_10_std": -0.479983, + "nauc_recall_at_10_diff1": 0.641467, + "nauc_recall_at_20_max": 0.585835, + "nauc_recall_at_20_std": -0.545451, + "nauc_recall_at_20_diff1": 0.640002, + "nauc_recall_at_100_max": 0.508858, + "nauc_recall_at_100_std": -0.835249, + "nauc_recall_at_100_diff1": 0.728942, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.524056, + "nauc_precision_at_1_std": -0.4011, + "nauc_precision_at_1_diff1": 0.783681, + "nauc_precision_at_3_max": 0.498365, + "nauc_precision_at_3_std": -0.462726, + "nauc_precision_at_3_diff1": 0.596299, + "nauc_precision_at_5_max": 0.48543, + "nauc_precision_at_5_std": -0.423947, + "nauc_precision_at_5_diff1": 0.530296, + "nauc_precision_at_10_max": 0.457112, + "nauc_precision_at_10_std": -0.385217, + "nauc_precision_at_10_diff1": 0.434993, + "nauc_precision_at_20_max": 0.372965, + "nauc_precision_at_20_std": -0.351837, + "nauc_precision_at_20_diff1": 0.334401, + "nauc_precision_at_100_max": 0.080201, + "nauc_precision_at_100_std": -0.165989, + "nauc_precision_at_100_diff1": 0.033533, + "nauc_precision_at_1000_max": -0.07857, + "nauc_precision_at_1000_std": 0.111515, + "nauc_precision_at_1000_diff1": -0.198888, + "nauc_mrr_at_1_max": 0.524056, + "nauc_mrr_at_1_std": -0.4011, + "nauc_mrr_at_1_diff1": 0.783681, + "nauc_mrr_at_3_max": 0.542513, + "nauc_mrr_at_3_std": -0.441941, + "nauc_mrr_at_3_diff1": 0.762136, + "nauc_mrr_at_5_max": 0.544413, + "nauc_mrr_at_5_std": -0.43865, + "nauc_mrr_at_5_diff1": 0.758728, + "nauc_mrr_at_10_max": 0.546418, + "nauc_mrr_at_10_std": -0.438166, + "nauc_mrr_at_10_diff1": 0.758043, + "nauc_mrr_at_20_max": 0.544737, + "nauc_mrr_at_20_std": -0.439672, + "nauc_mrr_at_20_diff1": 0.758595, + "nauc_mrr_at_100_max": 0.54385, + "nauc_mrr_at_100_std": -0.439155, + "nauc_mrr_at_100_diff1": 0.759457, + "nauc_mrr_at_1000_max": 0.543892, + "nauc_mrr_at_1000_std": -0.4389, + "nauc_mrr_at_1000_diff1": 0.759457, + "hit_rate_at_1": 0.63136, + "hit_rate_at_3": 0.75805, + "hit_rate_at_5": 0.79855, + "hit_rate_at_10": 0.84528, + "hit_rate_at_20": 0.88577, + "hit_rate_at_100": 0.96989, + "hit_rate_at_1000": 1.0, + "main_score": 0.7299, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.81388, + "ndcg_at_3": 0.87749, + "ndcg_at_5": 0.8901, + "ndcg_at_10": 0.89973, + "ndcg_at_20": 0.90291, + "ndcg_at_100": 0.90456, + "ndcg_at_1000": 0.90456, + "map_at_1": 0.79209, + "map_at_3": 0.85935, + "map_at_5": 0.86752, + "map_at_10": 0.87182, + "map_at_20": 0.87276, + "map_at_100": 0.87309, + "map_at_1000": 0.87309, + "recall_at_1": 0.79209, + "recall_at_3": 0.9217, + "recall_at_5": 0.95096, + "recall_at_10": 0.98013, + "recall_at_20": 0.9922, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "accuracy": 0.79209, + "precision_at_1": 0.81388, + "precision_at_3": 0.3226, + "precision_at_5": 0.2008, + "precision_at_10": 0.10372, + "precision_at_20": 0.05252, + "precision_at_100": 0.0106, + "precision_at_1000": 0.00106, + "mrr_at_1": 0.813883, + "mrr_at_3": 0.866365, + "mrr_at_5": 0.872351, + "mrr_at_10": 0.875937, + "mrr_at_20": 0.876826, + "mrr_at_100": 0.877017, + "mrr_at_1000": 0.877017, + "nauc_ndcg_at_1_max": 0.488167, + "nauc_ndcg_at_1_std": -0.430763, + "nauc_ndcg_at_1_diff1": 0.840827, + "nauc_ndcg_at_3_max": 0.504034, + "nauc_ndcg_at_3_std": -0.474146, + "nauc_ndcg_at_3_diff1": 0.79347, + "nauc_ndcg_at_5_max": 0.507679, + "nauc_ndcg_at_5_std": -0.44686, + "nauc_ndcg_at_5_diff1": 0.797158, + "nauc_ndcg_at_10_max": 0.497733, + "nauc_ndcg_at_10_std": -0.459702, + "nauc_ndcg_at_10_diff1": 0.803472, + "nauc_ndcg_at_20_max": 0.501041, + "nauc_ndcg_at_20_std": -0.447313, + "nauc_ndcg_at_20_diff1": 0.80876, + "nauc_ndcg_at_100_max": 0.500978, + "nauc_ndcg_at_100_std": -0.443437, + "nauc_ndcg_at_100_diff1": 0.810013, + "nauc_ndcg_at_1000_max": 0.500978, + "nauc_ndcg_at_1000_std": -0.443437, + "nauc_ndcg_at_1000_diff1": 0.810013, + "nauc_map_at_1_max": 0.45408, + "nauc_map_at_1_std": -0.429802, + "nauc_map_at_1_diff1": 0.835467, + "nauc_map_at_3_max": 0.491831, + "nauc_map_at_3_std": -0.464546, + "nauc_map_at_3_diff1": 0.805403, + "nauc_map_at_5_max": 0.497154, + "nauc_map_at_5_std": -0.448484, + "nauc_map_at_5_diff1": 0.808825, + "nauc_map_at_10_max": 0.49434, + "nauc_map_at_10_std": -0.450095, + "nauc_map_at_10_diff1": 0.810469, + "nauc_map_at_20_max": 0.49506, + "nauc_map_at_20_std": -0.44665, + "nauc_map_at_20_diff1": 0.811714, + "nauc_map_at_100_max": 0.495205, + "nauc_map_at_100_std": -0.445888, + "nauc_map_at_100_diff1": 0.81185, + "nauc_map_at_1000_max": 0.495205, + "nauc_map_at_1000_std": -0.445888, + "nauc_map_at_1000_diff1": 0.81185, + "nauc_recall_at_1_max": 0.45408, + "nauc_recall_at_1_std": -0.429802, + "nauc_recall_at_1_diff1": 0.835467, + "nauc_recall_at_3_max": 0.52824, + "nauc_recall_at_3_std": -0.551805, + "nauc_recall_at_3_diff1": 0.725584, + "nauc_recall_at_5_max": 0.54634, + "nauc_recall_at_5_std": -0.470439, + "nauc_recall_at_5_diff1": 0.702299, + "nauc_recall_at_10_max": 0.448115, + "nauc_recall_at_10_std": -0.727703, + "nauc_recall_at_10_diff1": 0.679469, + "nauc_recall_at_20_max": 0.520685, + "nauc_recall_at_20_std": -0.646069, + "nauc_recall_at_20_diff1": 0.727752, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.488167, + "nauc_precision_at_1_std": -0.430763, + "nauc_precision_at_1_diff1": 0.840827, + "nauc_precision_at_3_max": 0.274483, + "nauc_precision_at_3_std": -0.209926, + "nauc_precision_at_3_diff1": 0.192747, + "nauc_precision_at_5_max": 0.187585, + "nauc_precision_at_5_std": -0.01749, + "nauc_precision_at_5_diff1": 0.000586, + "nauc_precision_at_10_max": 0.05111, + "nauc_precision_at_10_std": 0.100324, + "nauc_precision_at_10_diff1": -0.215308, + "nauc_precision_at_20_max": 0.006212, + "nauc_precision_at_20_std": 0.214684, + "nauc_precision_at_20_diff1": -0.31946, + "nauc_precision_at_100_max": -0.038371, + "nauc_precision_at_100_std": 0.281189, + "nauc_precision_at_100_diff1": -0.381864, + "nauc_precision_at_1000_max": -0.038371, + "nauc_precision_at_1000_std": 0.281189, + "nauc_precision_at_1000_diff1": -0.381864, + "nauc_mrr_at_1_max": 0.488167, + "nauc_mrr_at_1_std": -0.430763, + "nauc_mrr_at_1_diff1": 0.840827, + "nauc_mrr_at_3_max": 0.511255, + "nauc_mrr_at_3_std": -0.446005, + "nauc_mrr_at_3_diff1": 0.811942, + "nauc_mrr_at_5_max": 0.507719, + "nauc_mrr_at_5_std": -0.43362, + "nauc_mrr_at_5_diff1": 0.812533, + "nauc_mrr_at_10_max": 0.504015, + "nauc_mrr_at_10_std": -0.440323, + "nauc_mrr_at_10_diff1": 0.815659, + "nauc_mrr_at_20_max": 0.504769, + "nauc_mrr_at_20_std": -0.437434, + "nauc_mrr_at_20_diff1": 0.817047, + "nauc_mrr_at_100_max": 0.504576, + "nauc_mrr_at_100_std": -0.437459, + "nauc_mrr_at_100_diff1": 0.817076, + "nauc_mrr_at_1000_max": 0.504576, + "nauc_mrr_at_1000_std": -0.437459, + "nauc_mrr_at_1000_diff1": 0.817076, + "hit_rate_at_1": 0.81388, + "hit_rate_at_3": 0.92958, + "hit_rate_at_5": 0.95473, + "hit_rate_at_10": 0.98189, + "hit_rate_at_20": 0.99396, + "hit_rate_at_100": 1.0, + "hit_rate_at_1000": 1.0, + "main_score": 0.89973, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ] + }, + { + "ndcg_at_1": 0.47242, + "ndcg_at_3": 0.54135, + "ndcg_at_5": 0.57025, + "ndcg_at_10": 0.59639, + "ndcg_at_20": 0.61447, + "ndcg_at_100": 0.6355, + "ndcg_at_1000": 0.64698, + "map_at_1": 0.44953, + "map_at_3": 0.5187, + "map_at_5": 0.53539, + "map_at_10": 0.54669, + "map_at_20": 0.55198, + "map_at_100": 0.55518, + "map_at_1000": 0.55569, + "recall_at_1": 0.44953, + "recall_at_3": 0.59105, + "recall_at_5": 0.65869, + "recall_at_10": 0.7376, + "recall_at_20": 0.8061, + "recall_at_100": 0.91467, + "recall_at_1000": 1.0, + "accuracy": 0.44953, + "precision_at_1": 0.47242, + "precision_at_3": 0.21263, + "precision_at_5": 0.14318, + "precision_at_10": 0.08033, + "precision_at_20": 0.04422, + "precision_at_100": 0.01011, + "precision_at_1000": 0.00111, + "mrr_at_1": 0.472425, + "mrr_at_3": 0.534686, + "mrr_at_5": 0.550451, + "mrr_at_10": 0.560577, + "mrr_at_20": 0.565248, + "mrr_at_100": 0.567864, + "mrr_at_1000": 0.568301, + "nauc_ndcg_at_1_max": 0.359046, + "nauc_ndcg_at_1_std": -0.014526, + "nauc_ndcg_at_1_diff1": 0.650857, + "nauc_ndcg_at_3_max": 0.36441, + "nauc_ndcg_at_3_std": -0.030047, + "nauc_ndcg_at_3_diff1": 0.603196, + "nauc_ndcg_at_5_max": 0.369732, + "nauc_ndcg_at_5_std": -0.027654, + "nauc_ndcg_at_5_diff1": 0.591639, + "nauc_ndcg_at_10_max": 0.363026, + "nauc_ndcg_at_10_std": -0.028587, + "nauc_ndcg_at_10_diff1": 0.591004, + "nauc_ndcg_at_20_max": 0.372489, + "nauc_ndcg_at_20_std": -0.018554, + "nauc_ndcg_at_20_diff1": 0.591813, + "nauc_ndcg_at_100_max": 0.367361, + "nauc_ndcg_at_100_std": -0.019256, + "nauc_ndcg_at_100_diff1": 0.5971, + "nauc_ndcg_at_1000_max": 0.367741, + "nauc_ndcg_at_1000_std": -0.021585, + "nauc_ndcg_at_1000_diff1": 0.601099, + "nauc_map_at_1_max": 0.33566, + "nauc_map_at_1_std": -0.018375, + "nauc_map_at_1_diff1": 0.647661, + "nauc_map_at_3_max": 0.362883, + "nauc_map_at_3_std": -0.02688, + "nauc_map_at_3_diff1": 0.611872, + "nauc_map_at_5_max": 0.366754, + "nauc_map_at_5_std": -0.024849, + "nauc_map_at_5_diff1": 0.60693, + "nauc_map_at_10_max": 0.363691, + "nauc_map_at_10_std": -0.025679, + "nauc_map_at_10_diff1": 0.606964, + "nauc_map_at_20_max": 0.366453, + "nauc_map_at_20_std": -0.022595, + "nauc_map_at_20_diff1": 0.607354, + "nauc_map_at_100_max": 0.365656, + "nauc_map_at_100_std": -0.022681, + "nauc_map_at_100_diff1": 0.607917, + "nauc_map_at_1000_max": 0.365736, + "nauc_map_at_1000_std": -0.022657, + "nauc_map_at_1000_diff1": 0.60804, + "nauc_recall_at_1_max": 0.33566, + "nauc_recall_at_1_std": -0.018375, + "nauc_recall_at_1_diff1": 0.647661, + "nauc_recall_at_3_max": 0.363444, + "nauc_recall_at_3_std": -0.036049, + "nauc_recall_at_3_diff1": 0.562648, + "nauc_recall_at_5_max": 0.373288, + "nauc_recall_at_5_std": -0.031841, + "nauc_recall_at_5_diff1": 0.522613, + "nauc_recall_at_10_max": 0.350165, + "nauc_recall_at_10_std": -0.031455, + "nauc_recall_at_10_diff1": 0.505989, + "nauc_recall_at_20_max": 0.404189, + "nauc_recall_at_20_std": 0.023574, + "nauc_recall_at_20_diff1": 0.491124, + "nauc_recall_at_100_max": 0.367011, + "nauc_recall_at_100_std": 0.056286, + "nauc_recall_at_100_diff1": 0.490337, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.359046, + "nauc_precision_at_1_std": -0.014526, + "nauc_precision_at_1_diff1": 0.650857, + "nauc_precision_at_3_max": 0.371761, + "nauc_precision_at_3_std": -0.022617, + "nauc_precision_at_3_diff1": 0.485239, + "nauc_precision_at_5_max": 0.366695, + "nauc_precision_at_5_std": -0.014189, + "nauc_precision_at_5_diff1": 0.416028, + "nauc_precision_at_10_max": 0.313346, + "nauc_precision_at_10_std": -0.020858, + "nauc_precision_at_10_diff1": 0.352847, + "nauc_precision_at_20_max": 0.313722, + "nauc_precision_at_20_std": 0.031477, + "nauc_precision_at_20_diff1": 0.269234, + "nauc_precision_at_100_max": 0.161808, + "nauc_precision_at_100_std": 0.03158, + "nauc_precision_at_100_diff1": 0.084324, + "nauc_precision_at_1000_max": 0.028547, + "nauc_precision_at_1000_std": 0.018903, + "nauc_precision_at_1000_diff1": -0.14149, + "nauc_mrr_at_1_max": 0.359046, + "nauc_mrr_at_1_std": -0.014526, + "nauc_mrr_at_1_diff1": 0.650857, + "nauc_mrr_at_3_max": 0.363866, + "nauc_mrr_at_3_std": -0.030414, + "nauc_mrr_at_3_diff1": 0.621047, + "nauc_mrr_at_5_max": 0.365944, + "nauc_mrr_at_5_std": -0.029118, + "nauc_mrr_at_5_diff1": 0.614153, + "nauc_mrr_at_10_max": 0.36333, + "nauc_mrr_at_10_std": -0.029543, + "nauc_mrr_at_10_diff1": 0.61355, + "nauc_mrr_at_20_max": 0.364961, + "nauc_mrr_at_20_std": -0.027382, + "nauc_mrr_at_20_diff1": 0.613631, + "nauc_mrr_at_100_max": 0.364429, + "nauc_mrr_at_100_std": -0.02717, + "nauc_mrr_at_100_diff1": 0.614054, + "nauc_mrr_at_1000_max": 0.364471, + "nauc_mrr_at_1000_std": -0.027177, + "nauc_mrr_at_1000_diff1": 0.614185, + "hit_rate_at_1": 0.47242, + "hit_rate_at_3": 0.61082, + "hit_rate_at_5": 0.6795, + "hit_rate_at_10": 0.75546, + "hit_rate_at_20": 0.82206, + "hit_rate_at_100": 0.92092, + "hit_rate_at_1000": 1.0, + "main_score": 0.59639, + "hf_subset": "sw", + "languages": [ + "swa-Latn" + ] + }, + { + "ndcg_at_1": 0.71333, + "ndcg_at_3": 0.78945, + "ndcg_at_5": 0.80567, + "ndcg_at_10": 0.82112, + "ndcg_at_20": 0.82914, + "ndcg_at_100": 0.83624, + "ndcg_at_1000": 0.83713, + "map_at_1": 0.69426, + "map_at_3": 0.76889, + "map_at_5": 0.77836, + "map_at_10": 0.7852, + "map_at_20": 0.78747, + "map_at_100": 0.78864, + "map_at_1000": 0.7887, + "recall_at_1": 0.69426, + "recall_at_3": 0.84241, + "recall_at_5": 0.88074, + "recall_at_10": 0.92722, + "recall_at_20": 0.95833, + "recall_at_100": 0.99389, + "recall_at_1000": 1.0, + "accuracy": 0.69426, + "precision_at_1": 0.71333, + "precision_at_3": 0.2937, + "precision_at_5": 0.18467, + "precision_at_10": 0.09744, + "precision_at_20": 0.05033, + "precision_at_100": 0.01047, + "precision_at_1000": 0.00105, + "mrr_at_1": 0.713333, + "mrr_at_3": 0.775185, + "mrr_at_5": 0.78363, + "mrr_at_10": 0.789468, + "mrr_at_20": 0.791688, + "mrr_at_100": 0.792718, + "mrr_at_1000": 0.792761, + "nauc_ndcg_at_1_max": 0.364267, + "nauc_ndcg_at_1_std": -0.34884, + "nauc_ndcg_at_1_diff1": 0.743704, + "nauc_ndcg_at_3_max": 0.365139, + "nauc_ndcg_at_3_std": -0.429798, + "nauc_ndcg_at_3_diff1": 0.721856, + "nauc_ndcg_at_5_max": 0.366447, + "nauc_ndcg_at_5_std": -0.435454, + "nauc_ndcg_at_5_diff1": 0.72041, + "nauc_ndcg_at_10_max": 0.384065, + "nauc_ndcg_at_10_std": -0.413489, + "nauc_ndcg_at_10_diff1": 0.721283, + "nauc_ndcg_at_20_max": 0.384592, + "nauc_ndcg_at_20_std": -0.403523, + "nauc_ndcg_at_20_diff1": 0.722259, + "nauc_ndcg_at_100_max": 0.378089, + "nauc_ndcg_at_100_std": -0.398089, + "nauc_ndcg_at_100_diff1": 0.726167, + "nauc_ndcg_at_1000_max": 0.374839, + "nauc_ndcg_at_1000_std": -0.402172, + "nauc_ndcg_at_1000_diff1": 0.725438, + "nauc_map_at_1_max": 0.326379, + "nauc_map_at_1_std": -0.338103, + "nauc_map_at_1_diff1": 0.732982, + "nauc_map_at_3_max": 0.359431, + "nauc_map_at_3_std": -0.410588, + "nauc_map_at_3_diff1": 0.726061, + "nauc_map_at_5_max": 0.361872, + "nauc_map_at_5_std": -0.411126, + "nauc_map_at_5_diff1": 0.725612, + "nauc_map_at_10_max": 0.369558, + "nauc_map_at_10_std": -0.402233, + "nauc_map_at_10_diff1": 0.725953, + "nauc_map_at_20_max": 0.369456, + "nauc_map_at_20_std": -0.399969, + "nauc_map_at_20_diff1": 0.726258, + "nauc_map_at_100_max": 0.368268, + "nauc_map_at_100_std": -0.399355, + "nauc_map_at_100_diff1": 0.726818, + "nauc_map_at_1000_max": 0.368111, + "nauc_map_at_1000_std": -0.399548, + "nauc_map_at_1000_diff1": 0.72678, + "nauc_recall_at_1_max": 0.326379, + "nauc_recall_at_1_std": -0.338103, + "nauc_recall_at_1_diff1": 0.732982, + "nauc_recall_at_3_max": 0.363308, + "nauc_recall_at_3_std": -0.514598, + "nauc_recall_at_3_diff1": 0.70013, + "nauc_recall_at_5_max": 0.36393, + "nauc_recall_at_5_std": -0.562807, + "nauc_recall_at_5_diff1": 0.69002, + "nauc_recall_at_10_max": 0.474466, + "nauc_recall_at_10_std": -0.489772, + "nauc_recall_at_10_diff1": 0.679204, + "nauc_recall_at_20_max": 0.55136, + "nauc_recall_at_20_std": -0.400853, + "nauc_recall_at_20_diff1": 0.66206, + "nauc_recall_at_100_max": 0.901961, + "nauc_recall_at_100_std": 0.278966, + "nauc_recall_at_100_diff1": 0.826203, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.364267, + "nauc_precision_at_1_std": -0.34884, + "nauc_precision_at_1_diff1": 0.743704, + "nauc_precision_at_3_max": 0.332151, + "nauc_precision_at_3_std": -0.374254, + "nauc_precision_at_3_diff1": 0.465114, + "nauc_precision_at_5_max": 0.311506, + "nauc_precision_at_5_std": -0.339988, + "nauc_precision_at_5_diff1": 0.370741, + "nauc_precision_at_10_max": 0.323432, + "nauc_precision_at_10_std": -0.169995, + "nauc_precision_at_10_diff1": 0.209076, + "nauc_precision_at_20_max": 0.304333, + "nauc_precision_at_20_std": -0.038059, + "nauc_precision_at_20_diff1": 0.06703, + "nauc_precision_at_100_max": 0.172451, + "nauc_precision_at_100_std": 0.180133, + "nauc_precision_at_100_diff1": -0.181742, + "nauc_precision_at_1000_max": 0.113798, + "nauc_precision_at_1000_std": 0.165476, + "nauc_precision_at_1000_diff1": -0.247568, + "nauc_mrr_at_1_max": 0.364267, + "nauc_mrr_at_1_std": -0.34884, + "nauc_mrr_at_1_diff1": 0.743704, + "nauc_mrr_at_3_max": 0.376607, + "nauc_mrr_at_3_std": -0.401451, + "nauc_mrr_at_3_diff1": 0.728675, + "nauc_mrr_at_5_max": 0.374442, + "nauc_mrr_at_5_std": -0.405863, + "nauc_mrr_at_5_diff1": 0.728023, + "nauc_mrr_at_10_max": 0.378484, + "nauc_mrr_at_10_std": -0.398794, + "nauc_mrr_at_10_diff1": 0.728428, + "nauc_mrr_at_20_max": 0.378427, + "nauc_mrr_at_20_std": -0.396571, + "nauc_mrr_at_20_diff1": 0.728804, + "nauc_mrr_at_100_max": 0.377212, + "nauc_mrr_at_100_std": -0.396297, + "nauc_mrr_at_100_diff1": 0.729188, + "nauc_mrr_at_1000_max": 0.377094, + "nauc_mrr_at_1000_std": -0.396447, + "nauc_mrr_at_1000_diff1": 0.729161, + "hit_rate_at_1": 0.71333, + "hit_rate_at_3": 0.85, + "hit_rate_at_5": 0.88667, + "hit_rate_at_10": 0.93, + "hit_rate_at_20": 0.96111, + "hit_rate_at_100": 0.99444, + "hit_rate_at_1000": 1.0, + "main_score": 0.82112, + "hf_subset": "hi", + "languages": [ + "hin-Deva" + ] + }, + { + "ndcg_at_1": 0.67926, + "ndcg_at_3": 0.77162, + "ndcg_at_5": 0.79052, + "ndcg_at_10": 0.80755, + "ndcg_at_20": 0.81574, + "ndcg_at_100": 0.82274, + "ndcg_at_1000": 0.82359, + "map_at_1": 0.65339, + "map_at_3": 0.74444, + "map_at_5": 0.75609, + "map_at_10": 0.76398, + "map_at_20": 0.76652, + "map_at_100": 0.76773, + "map_at_1000": 0.76777, + "recall_at_1": 0.65339, + "recall_at_3": 0.83538, + "recall_at_5": 0.8793, + "recall_at_10": 0.92884, + "recall_at_20": 0.95948, + "recall_at_100": 0.9937, + "recall_at_1000": 1.0, + "accuracy": 0.65339, + "precision_at_1": 0.67926, + "precision_at_3": 0.29826, + "precision_at_5": 0.18958, + "precision_at_10": 0.10072, + "precision_at_20": 0.05209, + "precision_at_100": 0.01084, + "precision_at_1000": 0.00109, + "mrr_at_1": 0.679265, + "mrr_at_3": 0.758767, + "mrr_at_5": 0.767501, + "mrr_at_10": 0.7738, + "mrr_at_20": 0.775873, + "mrr_at_100": 0.77685, + "mrr_at_1000": 0.776876, + "nauc_ndcg_at_1_max": 0.348078, + "nauc_ndcg_at_1_std": -0.296143, + "nauc_ndcg_at_1_diff1": 0.719526, + "nauc_ndcg_at_3_max": 0.381195, + "nauc_ndcg_at_3_std": -0.351672, + "nauc_ndcg_at_3_diff1": 0.69079, + "nauc_ndcg_at_5_max": 0.378678, + "nauc_ndcg_at_5_std": -0.360043, + "nauc_ndcg_at_5_diff1": 0.683704, + "nauc_ndcg_at_10_max": 0.374911, + "nauc_ndcg_at_10_std": -0.368495, + "nauc_ndcg_at_10_diff1": 0.687333, + "nauc_ndcg_at_20_max": 0.373562, + "nauc_ndcg_at_20_std": -0.355104, + "nauc_ndcg_at_20_diff1": 0.689387, + "nauc_ndcg_at_100_max": 0.373736, + "nauc_ndcg_at_100_std": -0.339291, + "nauc_ndcg_at_100_diff1": 0.69444, + "nauc_ndcg_at_1000_max": 0.371647, + "nauc_ndcg_at_1000_std": -0.341229, + "nauc_ndcg_at_1000_diff1": 0.694665, + "nauc_map_at_1_max": 0.299503, + "nauc_map_at_1_std": -0.291416, + "nauc_map_at_1_diff1": 0.724706, + "nauc_map_at_3_max": 0.365716, + "nauc_map_at_3_std": -0.335191, + "nauc_map_at_3_diff1": 0.701923, + "nauc_map_at_5_max": 0.365168, + "nauc_map_at_5_std": -0.337571, + "nauc_map_at_5_diff1": 0.697994, + "nauc_map_at_10_max": 0.363242, + "nauc_map_at_10_std": -0.341828, + "nauc_map_at_10_diff1": 0.698942, + "nauc_map_at_20_max": 0.362916, + "nauc_map_at_20_std": -0.338035, + "nauc_map_at_20_diff1": 0.699791, + "nauc_map_at_100_max": 0.363012, + "nauc_map_at_100_std": -0.335717, + "nauc_map_at_100_diff1": 0.700368, + "nauc_map_at_1000_max": 0.362954, + "nauc_map_at_1000_std": -0.335744, + "nauc_map_at_1000_diff1": 0.70038, + "nauc_recall_at_1_max": 0.299503, + "nauc_recall_at_1_std": -0.291416, + "nauc_recall_at_1_diff1": 0.724706, + "nauc_recall_at_3_max": 0.418498, + "nauc_recall_at_3_std": -0.411567, + "nauc_recall_at_3_diff1": 0.648721, + "nauc_recall_at_5_max": 0.421506, + "nauc_recall_at_5_std": -0.467474, + "nauc_recall_at_5_diff1": 0.616637, + "nauc_recall_at_10_max": 0.422043, + "nauc_recall_at_10_std": -0.591585, + "nauc_recall_at_10_diff1": 0.604045, + "nauc_recall_at_20_max": 0.435562, + "nauc_recall_at_20_std": -0.570149, + "nauc_recall_at_20_diff1": 0.573267, + "nauc_recall_at_100_max": 0.767054, + "nauc_recall_at_100_std": 0.06759, + "nauc_recall_at_100_diff1": 0.665337, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.348078, + "nauc_precision_at_1_std": -0.296143, + "nauc_precision_at_1_diff1": 0.719526, + "nauc_precision_at_3_max": 0.373856, + "nauc_precision_at_3_std": -0.244344, + "nauc_precision_at_3_diff1": 0.338719, + "nauc_precision_at_5_max": 0.31257, + "nauc_precision_at_5_std": -0.188219, + "nauc_precision_at_5_diff1": 0.187527, + "nauc_precision_at_10_max": 0.224817, + "nauc_precision_at_10_std": -0.135901, + "nauc_precision_at_10_diff1": 0.008985, + "nauc_precision_at_20_max": 0.169415, + "nauc_precision_at_20_std": -0.013301, + "nauc_precision_at_20_diff1": -0.11736, + "nauc_precision_at_100_max": 0.093318, + "nauc_precision_at_100_std": 0.18371, + "nauc_precision_at_100_diff1": -0.273883, + "nauc_precision_at_1000_max": 0.055042, + "nauc_precision_at_1000_std": 0.183285, + "nauc_precision_at_1000_diff1": -0.305083, + "nauc_mrr_at_1_max": 0.348078, + "nauc_mrr_at_1_std": -0.296143, + "nauc_mrr_at_1_diff1": 0.719526, + "nauc_mrr_at_3_max": 0.379492, + "nauc_mrr_at_3_std": -0.337025, + "nauc_mrr_at_3_diff1": 0.692489, + "nauc_mrr_at_5_max": 0.375527, + "nauc_mrr_at_5_std": -0.344032, + "nauc_mrr_at_5_diff1": 0.690902, + "nauc_mrr_at_10_max": 0.374705, + "nauc_mrr_at_10_std": -0.342616, + "nauc_mrr_at_10_diff1": 0.693998, + "nauc_mrr_at_20_max": 0.37404, + "nauc_mrr_at_20_std": -0.340185, + "nauc_mrr_at_20_diff1": 0.694363, + "nauc_mrr_at_100_max": 0.374039, + "nauc_mrr_at_100_std": -0.338567, + "nauc_mrr_at_100_diff1": 0.695025, + "nauc_mrr_at_1000_max": 0.373995, + "nauc_mrr_at_1000_std": -0.338588, + "nauc_mrr_at_1000_diff1": 0.695026, + "hit_rate_at_1": 0.67926, + "hit_rate_at_3": 0.85291, + "hit_rate_at_5": 0.89173, + "hit_rate_at_10": 0.93667, + "hit_rate_at_20": 0.96425, + "hit_rate_at_100": 0.99489, + "hit_rate_at_1000": 1.0, + "main_score": 0.80755, + "hf_subset": "ur", + "languages": [ + "urd-Arab" + ] + } + ] + }, + "evaluation_time": 1134.2390282154083, + "kg_co2_emissions": null, + "date": 1777978697.038874 +} \ No newline at end of file diff --git a/results/jinaai__jina-embeddings-v5-text-small/dd76d535f5447ca3897a9c893fb1e612ead98192/LexRetrieval.v1.json b/results/jinaai__jina-embeddings-v5-text-small/dd76d535f5447ca3897a9c893fb1e612ead98192/LexRetrieval.v1.json new file mode 100644 index 0000000000..1e191b4f1f --- /dev/null +++ b/results/jinaai__jina-embeddings-v5-text-small/dd76d535f5447ca3897a9c893fb1e612ead98192/LexRetrieval.v1.json @@ -0,0 +1,167 @@ +{ + "dataset_revision": "b5b22dee3a30bb32ac579c14afe9eb152e8c70c7", + "task_name": "LexRetrieval.v1", + "mteb_version": "2.12.37", + "scores": { + "test": [ + { + "ndcg_at_1": 0.93637, + "ndcg_at_3": 0.95718, + "ndcg_at_5": 0.96003, + "ndcg_at_10": 0.96188, + "ndcg_at_20": 0.96297, + "ndcg_at_100": 0.96454, + "ndcg_at_1000": 0.96496, + "map_at_1": 0.93637, + "map_at_3": 0.95244, + "map_at_5": 0.95405, + "map_at_10": 0.95481, + "map_at_20": 0.95512, + "map_at_100": 0.95534, + "map_at_1000": 0.95535, + "recall_at_1": 0.93637, + "recall_at_3": 0.97075, + "recall_at_5": 0.97758, + "recall_at_10": 0.98324, + "recall_at_20": 0.98751, + "recall_at_100": 0.99594, + "recall_at_1000": 0.99925, + "accuracy": 0.93637, + "precision_at_1": 0.93637, + "precision_at_3": 0.32358, + "precision_at_5": 0.19552, + "precision_at_10": 0.09832, + "precision_at_20": 0.04938, + "precision_at_100": 0.00996, + "precision_at_1000": 0.001, + "mrr_at_1": 0.936372, + "mrr_at_3": 0.952439, + "mrr_at_5": 0.954046, + "mrr_at_10": 0.954814, + "mrr_at_20": 0.955121, + "mrr_at_100": 0.955336, + "mrr_at_1000": 0.955353, + "nauc_ndcg_at_1_max": 0.79979, + "nauc_ndcg_at_1_std": 0.125725, + "nauc_ndcg_at_1_diff1": 0.948689, + "nauc_ndcg_at_3_max": 0.867585, + "nauc_ndcg_at_3_std": 0.160252, + "nauc_ndcg_at_3_diff1": 0.95613, + "nauc_ndcg_at_5_max": 0.863814, + "nauc_ndcg_at_5_std": 0.164979, + "nauc_ndcg_at_5_diff1": 0.954763, + "nauc_ndcg_at_10_max": 0.859472, + "nauc_ndcg_at_10_std": 0.161069, + "nauc_ndcg_at_10_diff1": 0.955905, + "nauc_ndcg_at_20_max": 0.856757, + "nauc_ndcg_at_20_std": 0.164912, + "nauc_ndcg_at_20_diff1": 0.95547, + "nauc_ndcg_at_100_max": 0.852159, + "nauc_ndcg_at_100_std": 0.164032, + "nauc_ndcg_at_100_diff1": 0.954719, + "nauc_ndcg_at_1000_max": 0.850483, + "nauc_ndcg_at_1000_std": 0.160492, + "nauc_ndcg_at_1000_diff1": 0.954375, + "nauc_map_at_1_max": 0.79979, + "nauc_map_at_1_std": 0.125725, + "nauc_map_at_1_diff1": 0.948689, + "nauc_map_at_3_max": 0.848932, + "nauc_map_at_3_std": 0.15086, + "nauc_map_at_3_diff1": 0.953962, + "nauc_map_at_5_max": 0.84643, + "nauc_map_at_5_std": 0.152817, + "nauc_map_at_5_diff1": 0.953206, + "nauc_map_at_10_max": 0.844645, + "nauc_map_at_10_std": 0.151369, + "nauc_map_at_10_diff1": 0.953666, + "nauc_map_at_20_max": 0.843915, + "nauc_map_at_20_std": 0.152045, + "nauc_map_at_20_diff1": 0.95356, + "nauc_map_at_100_max": 0.843373, + "nauc_map_at_100_std": 0.151881, + "nauc_map_at_100_diff1": 0.953472, + "nauc_map_at_1000_max": 0.84332, + "nauc_map_at_1000_std": 0.151811, + "nauc_map_at_1000_diff1": 0.953461, + "nauc_recall_at_1_max": 0.79979, + "nauc_recall_at_1_std": 0.125725, + "nauc_recall_at_1_diff1": 0.948689, + "nauc_recall_at_3_max": 0.952943, + "nauc_recall_at_3_std": 0.203146, + "nauc_recall_at_3_diff1": 0.966147, + "nauc_recall_at_5_max": 0.962887, + "nauc_recall_at_5_std": 0.236643, + "nauc_recall_at_5_diff1": 0.963456, + "nauc_recall_at_10_max": 0.965479, + "nauc_recall_at_10_std": 0.231632, + "nauc_recall_at_10_diff1": 0.973181, + "nauc_recall_at_20_max": 0.970253, + "nauc_recall_at_20_std": 0.304372, + "nauc_recall_at_20_diff1": 0.973744, + "nauc_recall_at_100_max": 0.98581, + "nauc_recall_at_100_std": 0.552026, + "nauc_recall_at_100_diff1": 0.97549, + "nauc_recall_at_1000_max": 0.960318, + "nauc_recall_at_1000_std": 0.853943, + "nauc_recall_at_1000_diff1": 0.941643, + "nauc_precision_at_1_max": 0.79979, + "nauc_precision_at_1_std": 0.125725, + "nauc_precision_at_1_diff1": 0.948689, + "nauc_precision_at_3_max": 0.952943, + "nauc_precision_at_3_std": 0.203146, + "nauc_precision_at_3_diff1": 0.966147, + "nauc_precision_at_5_max": 0.962887, + "nauc_precision_at_5_std": 0.236643, + "nauc_precision_at_5_diff1": 0.963456, + "nauc_precision_at_10_max": 0.965479, + "nauc_precision_at_10_std": 0.231632, + "nauc_precision_at_10_diff1": 0.973181, + "nauc_precision_at_20_max": 0.970253, + "nauc_precision_at_20_std": 0.304372, + "nauc_precision_at_20_diff1": 0.973744, + "nauc_precision_at_100_max": 0.98581, + "nauc_precision_at_100_std": 0.552026, + "nauc_precision_at_100_diff1": 0.97549, + "nauc_precision_at_1000_max": 0.960318, + "nauc_precision_at_1000_std": 0.853943, + "nauc_precision_at_1000_diff1": 0.941643, + "nauc_mrr_at_1_max": 0.79979, + "nauc_mrr_at_1_std": 0.125725, + "nauc_mrr_at_1_diff1": 0.948689, + "nauc_mrr_at_3_max": 0.848932, + "nauc_mrr_at_3_std": 0.15086, + "nauc_mrr_at_3_diff1": 0.953962, + "nauc_mrr_at_5_max": 0.84643, + "nauc_mrr_at_5_std": 0.152817, + "nauc_mrr_at_5_diff1": 0.953206, + "nauc_mrr_at_10_max": 0.844645, + "nauc_mrr_at_10_std": 0.151369, + "nauc_mrr_at_10_diff1": 0.953666, + "nauc_mrr_at_20_max": 0.843915, + "nauc_mrr_at_20_std": 0.152045, + "nauc_mrr_at_20_diff1": 0.95356, + "nauc_mrr_at_100_max": 0.843373, + "nauc_mrr_at_100_std": 0.151881, + "nauc_mrr_at_100_diff1": 0.953472, + "nauc_mrr_at_1000_max": 0.84332, + "nauc_mrr_at_1000_std": 0.151811, + "nauc_mrr_at_1000_diff1": 0.953461, + "hit_rate_at_1": 0.93637, + "hit_rate_at_3": 0.97075, + "hit_rate_at_5": 0.97758, + "hit_rate_at_10": 0.98324, + "hit_rate_at_20": 0.98751, + "hit_rate_at_100": 0.99594, + "hit_rate_at_1000": 0.99925, + "main_score": 0.96188, + "hf_subset": "default", + "languages": [ + "dan-Latn" + ] + } + ] + }, + "evaluation_time": 647.7694940567017, + "kg_co2_emissions": null, + "date": 1777967824.689914 +} \ No newline at end of file diff --git a/results/jinaai__jina-embeddings-v5-text-small/dd76d535f5447ca3897a9c893fb1e612ead98192/PublicNewsRetrieval.json b/results/jinaai__jina-embeddings-v5-text-small/dd76d535f5447ca3897a9c893fb1e612ead98192/PublicNewsRetrieval.json new file mode 100644 index 0000000000..6187d1a567 --- /dev/null +++ b/results/jinaai__jina-embeddings-v5-text-small/dd76d535f5447ca3897a9c893fb1e612ead98192/PublicNewsRetrieval.json @@ -0,0 +1,322 @@ +{ + "dataset_revision": "82dad8c469646feae9631d69ba92bebfc0a586e9", + "task_name": "PublicNewsRetrieval", + "mteb_version": "2.12.37", + "scores": { + "test": [ + { + "ndcg_at_1": 0.76743, + "ndcg_at_3": 0.81705, + "ndcg_at_5": 0.82996, + "ndcg_at_10": 0.84042, + "ndcg_at_20": 0.84651, + "ndcg_at_100": 0.85354, + "ndcg_at_1000": 0.85691, + "map_at_1": 0.75581, + "map_at_3": 0.80293, + "map_at_5": 0.81054, + "map_at_10": 0.81491, + "map_at_20": 0.81677, + "map_at_100": 0.81783, + "map_at_1000": 0.81797, + "recall_at_1": 0.75581, + "recall_at_3": 0.85143, + "recall_at_5": 0.88215, + "recall_at_10": 0.91382, + "recall_at_20": 0.93672, + "recall_at_100": 0.97336, + "recall_at_1000": 0.99866, + "accuracy": 0.75581, + "precision_at_1": 0.76743, + "precision_at_3": 0.29178, + "precision_at_5": 0.1819, + "precision_at_10": 0.0945, + "precision_at_20": 0.04866, + "precision_at_100": 0.01016, + "precision_at_1000": 0.00105, + "mrr_at_1": 0.767426, + "mrr_at_3": 0.808758, + "mrr_at_5": 0.815929, + "mrr_at_10": 0.820154, + "mrr_at_20": 0.821565, + "mrr_at_100": 0.822429, + "mrr_at_1000": 0.822554, + "nauc_ndcg_at_1_max": 0.518796, + "nauc_ndcg_at_1_std": -0.4396, + "nauc_ndcg_at_1_diff1": 0.846287, + "nauc_ndcg_at_3_max": 0.59641, + "nauc_ndcg_at_3_std": -0.418865, + "nauc_ndcg_at_3_diff1": 0.833133, + "nauc_ndcg_at_5_max": 0.60404, + "nauc_ndcg_at_5_std": -0.419008, + "nauc_ndcg_at_5_diff1": 0.832209, + "nauc_ndcg_at_10_max": 0.608993, + "nauc_ndcg_at_10_std": -0.396015, + "nauc_ndcg_at_10_diff1": 0.829059, + "nauc_ndcg_at_20_max": 0.607516, + "nauc_ndcg_at_20_std": -0.381548, + "nauc_ndcg_at_20_diff1": 0.830334, + "nauc_ndcg_at_100_max": 0.600561, + "nauc_ndcg_at_100_std": -0.390476, + "nauc_ndcg_at_100_diff1": 0.833657, + "nauc_ndcg_at_1000_max": 0.59399, + "nauc_ndcg_at_1000_std": -0.398639, + "nauc_ndcg_at_1000_diff1": 0.83353, + "nauc_map_at_1_max": 0.491191, + "nauc_map_at_1_std": -0.435533, + "nauc_map_at_1_diff1": 0.849462, + "nauc_map_at_3_max": 0.572528, + "nauc_map_at_3_std": -0.423899, + "nauc_map_at_3_diff1": 0.836421, + "nauc_map_at_5_max": 0.577111, + "nauc_map_at_5_std": -0.424185, + "nauc_map_at_5_diff1": 0.836183, + "nauc_map_at_10_max": 0.579255, + "nauc_map_at_10_std": -0.414493, + "nauc_map_at_10_diff1": 0.834928, + "nauc_map_at_20_max": 0.579407, + "nauc_map_at_20_std": -0.410401, + "nauc_map_at_20_diff1": 0.835233, + "nauc_map_at_100_max": 0.578429, + "nauc_map_at_100_std": -0.411428, + "nauc_map_at_100_diff1": 0.83572, + "nauc_map_at_1000_max": 0.578198, + "nauc_map_at_1000_std": -0.411695, + "nauc_map_at_1000_diff1": 0.8357, + "nauc_recall_at_1_max": 0.491191, + "nauc_recall_at_1_std": -0.435533, + "nauc_recall_at_1_diff1": 0.849462, + "nauc_recall_at_3_max": 0.67064, + "nauc_recall_at_3_std": -0.394594, + "nauc_recall_at_3_diff1": 0.817807, + "nauc_recall_at_5_max": 0.709555, + "nauc_recall_at_5_std": -0.396345, + "nauc_recall_at_5_diff1": 0.811305, + "nauc_recall_at_10_max": 0.754259, + "nauc_recall_at_10_std": -0.283376, + "nauc_recall_at_10_diff1": 0.784021, + "nauc_recall_at_20_max": 0.786975, + "nauc_recall_at_20_std": -0.1264, + "nauc_recall_at_20_diff1": 0.782063, + "nauc_recall_at_100_max": 0.858359, + "nauc_recall_at_100_std": -0.047698, + "nauc_recall_at_100_diff1": 0.817993, + "nauc_recall_at_1000_max": 0.93465, + "nauc_recall_at_1000_std": 0.275507, + "nauc_recall_at_1000_diff1": 0.638225, + "nauc_precision_at_1_max": 0.518796, + "nauc_precision_at_1_std": -0.4396, + "nauc_precision_at_1_diff1": 0.846287, + "nauc_precision_at_3_max": 0.608347, + "nauc_precision_at_3_std": -0.298087, + "nauc_precision_at_3_diff1": 0.606445, + "nauc_precision_at_5_max": 0.610422, + "nauc_precision_at_5_std": -0.249043, + "nauc_precision_at_5_diff1": 0.523585, + "nauc_precision_at_10_max": 0.597388, + "nauc_precision_at_10_std": -0.080353, + "nauc_precision_at_10_diff1": 0.37787, + "nauc_precision_at_20_max": 0.493737, + "nauc_precision_at_20_std": 0.105132, + "nauc_precision_at_20_diff1": 0.2133, + "nauc_precision_at_100_max": 0.267136, + "nauc_precision_at_100_std": 0.226502, + "nauc_precision_at_100_diff1": -0.074751, + "nauc_precision_at_1000_max": 0.003073, + "nauc_precision_at_1000_std": 0.269802, + "nauc_precision_at_1000_diff1": -0.381288, + "nauc_mrr_at_1_max": 0.518796, + "nauc_mrr_at_1_std": -0.4396, + "nauc_mrr_at_1_diff1": 0.846287, + "nauc_mrr_at_3_max": 0.578514, + "nauc_mrr_at_3_std": -0.427383, + "nauc_mrr_at_3_diff1": 0.836928, + "nauc_mrr_at_5_max": 0.580631, + "nauc_mrr_at_5_std": -0.42618, + "nauc_mrr_at_5_diff1": 0.836438, + "nauc_mrr_at_10_max": 0.581875, + "nauc_mrr_at_10_std": -0.416674, + "nauc_mrr_at_10_diff1": 0.835355, + "nauc_mrr_at_20_max": 0.580354, + "nauc_mrr_at_20_std": -0.414747, + "nauc_mrr_at_20_diff1": 0.835601, + "nauc_mrr_at_100_max": 0.579463, + "nauc_mrr_at_100_std": -0.415872, + "nauc_mrr_at_100_diff1": 0.835902, + "nauc_mrr_at_1000_max": 0.57927, + "nauc_mrr_at_1000_std": -0.416092, + "nauc_mrr_at_1000_diff1": 0.835893, + "hit_rate_at_1": 0.76743, + "hit_rate_at_3": 0.85858, + "hit_rate_at_5": 0.88941, + "hit_rate_at_10": 0.92091, + "hit_rate_at_20": 0.94102, + "hit_rate_at_100": 0.9752, + "hit_rate_at_1000": 0.99866, + "main_score": 0.84042, + "hf_subset": "pl", + "languages": [ + "pol-Latn" + ] + }, + { + "ndcg_at_1": 0.71064, + "ndcg_at_3": 0.78158, + "ndcg_at_5": 0.80231, + "ndcg_at_10": 0.81604, + "ndcg_at_20": 0.82427, + "ndcg_at_100": 0.83174, + "ndcg_at_1000": 0.83298, + "map_at_1": 0.69196, + "map_at_3": 0.76102, + "map_at_5": 0.77345, + "map_at_10": 0.77956, + "map_at_20": 0.78196, + "map_at_100": 0.7831, + "map_at_1000": 0.78316, + "recall_at_1": 0.69196, + "recall_at_3": 0.83168, + "recall_at_5": 0.88046, + "recall_at_10": 0.92143, + "recall_at_20": 0.95303, + "recall_at_100": 0.99117, + "recall_at_1000": 0.99976, + "accuracy": 0.69196, + "precision_at_1": 0.71064, + "precision_at_3": 0.29157, + "precision_at_5": 0.186, + "precision_at_10": 0.09787, + "precision_at_20": 0.05069, + "precision_at_100": 0.01059, + "precision_at_1000": 0.00107, + "mrr_at_1": 0.710638, + "mrr_at_3": 0.76974, + "mrr_at_5": 0.780118, + "mrr_at_10": 0.785247, + "mrr_at_20": 0.787408, + "mrr_at_100": 0.788377, + "mrr_at_1000": 0.788426, + "nauc_ndcg_at_1_max": 0.313479, + "nauc_ndcg_at_1_std": -0.472808, + "nauc_ndcg_at_1_diff1": 0.822656, + "nauc_ndcg_at_3_max": 0.312331, + "nauc_ndcg_at_3_std": -0.577425, + "nauc_ndcg_at_3_diff1": 0.804427, + "nauc_ndcg_at_5_max": 0.322284, + "nauc_ndcg_at_5_std": -0.571857, + "nauc_ndcg_at_5_diff1": 0.805022, + "nauc_ndcg_at_10_max": 0.32443, + "nauc_ndcg_at_10_std": -0.568426, + "nauc_ndcg_at_10_diff1": 0.805472, + "nauc_ndcg_at_20_max": 0.323106, + "nauc_ndcg_at_20_std": -0.568208, + "nauc_ndcg_at_20_diff1": 0.807471, + "nauc_ndcg_at_100_max": 0.32088, + "nauc_ndcg_at_100_std": -0.548889, + "nauc_ndcg_at_100_diff1": 0.807378, + "nauc_ndcg_at_1000_max": 0.321652, + "nauc_ndcg_at_1000_std": -0.548301, + "nauc_ndcg_at_1000_diff1": 0.807824, + "nauc_map_at_1_max": 0.294186, + "nauc_map_at_1_std": -0.45974, + "nauc_map_at_1_diff1": 0.823886, + "nauc_map_at_3_max": 0.310278, + "nauc_map_at_3_std": -0.548409, + "nauc_map_at_3_diff1": 0.808599, + "nauc_map_at_5_max": 0.316297, + "nauc_map_at_5_std": -0.546408, + "nauc_map_at_5_diff1": 0.809447, + "nauc_map_at_10_max": 0.317856, + "nauc_map_at_10_std": -0.543308, + "nauc_map_at_10_diff1": 0.809072, + "nauc_map_at_20_max": 0.317706, + "nauc_map_at_20_std": -0.54306, + "nauc_map_at_20_diff1": 0.809505, + "nauc_map_at_100_max": 0.317602, + "nauc_map_at_100_std": -0.540496, + "nauc_map_at_100_diff1": 0.8095, + "nauc_map_at_1000_max": 0.317669, + "nauc_map_at_1000_std": -0.54043, + "nauc_map_at_1000_diff1": 0.809527, + "nauc_recall_at_1_max": 0.294186, + "nauc_recall_at_1_std": -0.45974, + "nauc_recall_at_1_diff1": 0.823886, + "nauc_recall_at_3_max": 0.305007, + "nauc_recall_at_3_std": -0.68111, + "nauc_recall_at_3_diff1": 0.785074, + "nauc_recall_at_5_max": 0.336079, + "nauc_recall_at_5_std": -0.698495, + "nauc_recall_at_5_diff1": 0.783753, + "nauc_recall_at_10_max": 0.353484, + "nauc_recall_at_10_std": -0.750634, + "nauc_recall_at_10_diff1": 0.781652, + "nauc_recall_at_20_max": 0.35099, + "nauc_recall_at_20_std": -0.876547, + "nauc_recall_at_20_diff1": 0.796064, + "nauc_recall_at_100_max": 0.23644, + "nauc_recall_at_100_std": -0.575915, + "nauc_recall_at_100_diff1": 0.753508, + "nauc_recall_at_1000_max": -0.564004, + "nauc_recall_at_1000_std": -0.564004, + "nauc_recall_at_1000_diff1": 0.722188, + "nauc_precision_at_1_max": 0.313479, + "nauc_precision_at_1_std": -0.472808, + "nauc_precision_at_1_diff1": 0.822656, + "nauc_precision_at_3_max": 0.261069, + "nauc_precision_at_3_std": -0.509831, + "nauc_precision_at_3_diff1": 0.483645, + "nauc_precision_at_5_max": 0.24884, + "nauc_precision_at_5_std": -0.414257, + "nauc_precision_at_5_diff1": 0.339577, + "nauc_precision_at_10_max": 0.202457, + "nauc_precision_at_10_std": -0.28936, + "nauc_precision_at_10_diff1": 0.161961, + "nauc_precision_at_20_max": 0.151236, + "nauc_precision_at_20_std": -0.190453, + "nauc_precision_at_20_diff1": -0.002368, + "nauc_precision_at_100_max": 0.043219, + "nauc_precision_at_100_std": 0.160625, + "nauc_precision_at_100_diff1": -0.299674, + "nauc_precision_at_1000_max": 0.026245, + "nauc_precision_at_1000_std": 0.223494, + "nauc_precision_at_1000_diff1": -0.368374, + "nauc_mrr_at_1_max": 0.313479, + "nauc_mrr_at_1_std": -0.472808, + "nauc_mrr_at_1_diff1": 0.822656, + "nauc_mrr_at_3_max": 0.318348, + "nauc_mrr_at_3_std": -0.548684, + "nauc_mrr_at_3_diff1": 0.808378, + "nauc_mrr_at_5_max": 0.322519, + "nauc_mrr_at_5_std": -0.541009, + "nauc_mrr_at_5_diff1": 0.808498, + "nauc_mrr_at_10_max": 0.322101, + "nauc_mrr_at_10_std": -0.539781, + "nauc_mrr_at_10_diff1": 0.80922, + "nauc_mrr_at_20_max": 0.321698, + "nauc_mrr_at_20_std": -0.539176, + "nauc_mrr_at_20_diff1": 0.809813, + "nauc_mrr_at_100_max": 0.321323, + "nauc_mrr_at_100_std": -0.537467, + "nauc_mrr_at_100_diff1": 0.809801, + "nauc_mrr_at_1000_max": 0.321348, + "nauc_mrr_at_1000_std": -0.537464, + "nauc_mrr_at_1000_diff1": 0.809819, + "hit_rate_at_1": 0.71064, + "hit_rate_at_3": 0.84208, + "hit_rate_at_5": 0.88747, + "hit_rate_at_10": 0.92671, + "hit_rate_at_20": 0.95745, + "hit_rate_at_100": 0.99291, + "hit_rate_at_1000": 1.0, + "main_score": 0.81604, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ] + } + ] + }, + "evaluation_time": 3613.182503223419, + "kg_co2_emissions": null, + "date": 1777984043.48525 +} \ No newline at end of file diff --git a/results/jinaai__jina-embeddings-v5-text-small/dd76d535f5447ca3897a9c893fb1e612ead98192/model_meta.json b/results/jinaai__jina-embeddings-v5-text-small/dd76d535f5447ca3897a9c893fb1e612ead98192/model_meta.json new file mode 100644 index 0000000000..8e199012f5 --- /dev/null +++ b/results/jinaai__jina-embeddings-v5-text-small/dd76d535f5447ca3897a9c893fb1e612ead98192/model_meta.json @@ -0,0 +1,132 @@ +{ + "loader_kwargs": { + "trust_remote_code": true, + "model_prompts": { + "Retrieval": "retrieval", + "Clustering": "clustering", + "Classification": "classification", + "STS": "text-matching", + "PairClassification": "text-matching", + "BitextMining": "text-matching", + "MultilabelClassification": "classification", + "Reranking": "retrieval", + "Summarization": "text-matching", + "InstructionReranking": "retrieval" + } + }, + "name": "jinaai/jina-embeddings-v5-text-small", + "revision": "dd76d535f5447ca3897a9c893fb1e612ead98192", + "release_date": "2026-02-17", + "languages": [ + "afr-Latn", + "ara-Arab", + "aze-Latn", + "bel-Cyrl", + "bul-Cyrl", + "ben-Beng", + "cat-Latn", + "ceb-Latn", + "ces-Latn", + "cym-Latn", + "dan-Latn", + "deu-Latn", + "ell-Grek", + "eng-Latn", + "spa-Latn", + "est-Latn", + "eus-Latn", + "fas-Arab", + "fin-Latn", + "fra-Latn", + "glg-Latn", + "guj-Gujr", + "heb-Hebr", + "hin-Deva", + "hrv-Latn", + "hat-Latn", + "hun-Latn", + "hye-Armn", + "ind-Latn", + "isl-Latn", + "ita-Latn", + "jpn-Jpan", + "jav-Latn", + "kat-Geor", + "kaz-Cyrl", + "khm-Khmr", + "kan-Knda", + "kor-Hang", + "kir-Cyrl", + "lao-Laoo", + "lit-Latn", + "lav-Latn", + "mkd-Cyrl", + "mal-Mlym", + "mon-Cyrl", + "mar-Deva", + "msa-Latn", + "mya-Mymr", + "nep-Deva", + "nld-Latn", + "nor-Latn", + "nob-Latn", + "nno-Latn", + "pan-Guru", + "pol-Latn", + "por-Latn", + "que-Latn", + "ron-Latn", + "rus-Cyrl", + "sin-Sinh", + "slk-Latn", + "slv-Latn", + "swa-Latn", + "tam-Taml", + "tel-Telu", + "tha-Thai", + "tgl-Latn", + "tur-Latn", + "ukr-Cyrl", + "urd-Arab", + "vie-Latn", + "yor-Latn", + "zho-Hans" + ], + "n_parameters": 596049920, + "n_active_parameters_override": null, + "n_embedding_parameters": 155582464, + "memory_usage_mb": 1137.0, + "max_tokens": 32768.0, + "embed_dim": 1024, + "license": "cc-by-nc-4.0", + "open_weights": true, + "public_training_code": null, + "public_training_data": null, + "framework": [ + "Sentence Transformers", + "PyTorch", + "Transformers", + "safetensors" + ], + "reference": "https://huggingface.co/jinaai/jina-embeddings-v5-text-small", + "similarity_fn_name": "cosine", + "use_instructions": true, + "training_datasets": null, + "adapted_from": "Qwen/Qwen3-0.6B", + "superseded_by": null, + "modalities": [ + "text" + ], + "model_type": [ + "dense" + ], + "citation": "@misc{akram2026jinaembeddingsv5texttasktargetedembeddingdistillation,\n title={jina-embeddings-v5-text: Task-Targeted Embedding Distillation},\n author={Mohammad Kalim Akram and Saba Sturua and Nastia Havriushenko and Quentin Herreros and Michael G\u00fcnther and Maximilian Werk and Han Xiao},\n year={2026},\n eprint={2602.15547},\n archivePrefix={arXiv},\n primaryClass={cs.CL},\n url={https://arxiv.org/abs/2602.15547},\n}", + "contacts": null, + "experiment_kwargs": null, + "output_dtypes": null, + "extra_requirements_groups": [ + "peft" + ], + "loader": "JinaV5TextWrapper", + "is_cross_encoder": false +} \ No newline at end of file diff --git a/results/microsoft__harrier-oss-v1-0.6b/f9b9dc8d367d443f2479d27aa5d8d2850c0774ee/ElasticKBRetrieval.json b/results/microsoft__harrier-oss-v1-0.6b/f9b9dc8d367d443f2479d27aa5d8d2850c0774ee/ElasticKBRetrieval.json new file mode 100644 index 0000000000..07bf774860 --- /dev/null +++ b/results/microsoft__harrier-oss-v1-0.6b/f9b9dc8d367d443f2479d27aa5d8d2850c0774ee/ElasticKBRetrieval.json @@ -0,0 +1,324 @@ +{ + "dataset_revision": "21bdf1e024bf7c9f46720017559ce2f8c6116507", + "task_name": "ElasticKBRetrieval", + "mteb_version": "2.12.38", + "scores": { + "synthetic_test": [ + { + "ndcg_at_1": 0.84043, + "ndcg_at_3": 0.65495, + "ndcg_at_5": 0.60927, + "ndcg_at_10": 0.57945, + "ndcg_at_20": 0.57914, + "ndcg_at_100": 0.61361, + "ndcg_at_1000": 0.67292, + "map_at_1": 0.26348, + "map_at_3": 0.32409, + "map_at_5": 0.34264, + "map_at_10": 0.37175, + "map_at_20": 0.39969, + "map_at_100": 0.42728, + "map_at_1000": 0.43901, + "recall_at_1": 0.26348, + "recall_at_3": 0.33699, + "recall_at_5": 0.37363, + "recall_at_10": 0.44228, + "recall_at_20": 0.52991, + "recall_at_100": 0.68001, + "recall_at_1000": 0.86024, + "accuracy": 0.26348, + "precision_at_1": 0.84043, + "precision_at_3": 0.49291, + "precision_at_5": 0.37872, + "precision_at_10": 0.26755, + "precision_at_20": 0.19282, + "precision_at_100": 0.07048, + "precision_at_1000": 0.01261, + "mrr_at_1": 0.840426, + "mrr_at_3": 0.882979, + "mrr_at_5": 0.889096, + "mrr_at_10": 0.892657, + "mrr_at_20": 0.89352, + "mrr_at_100": 0.893762, + "mrr_at_1000": 0.893762, + "nauc_ndcg_at_1_max": 0.427805, + "nauc_ndcg_at_1_std": -0.282245, + "nauc_ndcg_at_1_diff1": 0.78418, + "nauc_ndcg_at_3_max": 0.248922, + "nauc_ndcg_at_3_std": -0.022484, + "nauc_ndcg_at_3_diff1": 0.302998, + "nauc_ndcg_at_5_max": 0.261421, + "nauc_ndcg_at_5_std": 0.002282, + "nauc_ndcg_at_5_diff1": 0.275058, + "nauc_ndcg_at_10_max": 0.302479, + "nauc_ndcg_at_10_std": 0.029925, + "nauc_ndcg_at_10_diff1": 0.353206, + "nauc_ndcg_at_20_max": 0.290298, + "nauc_ndcg_at_20_std": 0.01707, + "nauc_ndcg_at_20_diff1": 0.405351, + "nauc_ndcg_at_100_max": 0.356825, + "nauc_ndcg_at_100_std": 0.107018, + "nauc_ndcg_at_100_diff1": 0.426089, + "nauc_ndcg_at_1000_max": 0.330531, + "nauc_ndcg_at_1000_std": 0.108287, + "nauc_ndcg_at_1000_diff1": 0.370723, + "nauc_map_at_1_max": 0.206333, + "nauc_map_at_1_std": -0.154136, + "nauc_map_at_1_diff1": 0.55415, + "nauc_map_at_3_max": 0.228806, + "nauc_map_at_3_std": -0.125027, + "nauc_map_at_3_diff1": 0.537639, + "nauc_map_at_5_max": 0.240773, + "nauc_map_at_5_std": -0.104765, + "nauc_map_at_5_diff1": 0.513319, + "nauc_map_at_10_max": 0.267032, + "nauc_map_at_10_std": -0.063324, + "nauc_map_at_10_diff1": 0.491673, + "nauc_map_at_20_max": 0.293816, + "nauc_map_at_20_std": -0.015624, + "nauc_map_at_20_diff1": 0.468182, + "nauc_map_at_100_max": 0.315314, + "nauc_map_at_100_std": 0.033346, + "nauc_map_at_100_diff1": 0.448724, + "nauc_map_at_1000_max": 0.310964, + "nauc_map_at_1000_std": 0.035541, + "nauc_map_at_1000_diff1": 0.439555, + "nauc_recall_at_1_max": 0.206333, + "nauc_recall_at_1_std": -0.154136, + "nauc_recall_at_1_diff1": 0.55415, + "nauc_recall_at_3_max": 0.213259, + "nauc_recall_at_3_std": -0.125181, + "nauc_recall_at_3_diff1": 0.51964, + "nauc_recall_at_5_max": 0.209752, + "nauc_recall_at_5_std": -0.0789, + "nauc_recall_at_5_diff1": 0.446546, + "nauc_recall_at_10_max": 0.221841, + "nauc_recall_at_10_std": -0.024323, + "nauc_recall_at_10_diff1": 0.404991, + "nauc_recall_at_20_max": 0.228661, + "nauc_recall_at_20_std": 0.018172, + "nauc_recall_at_20_diff1": 0.357189, + "nauc_recall_at_100_max": 0.282334, + "nauc_recall_at_100_std": 0.202908, + "nauc_recall_at_100_diff1": 0.324845, + "nauc_recall_at_1000_max": 0.130199, + "nauc_recall_at_1000_std": 0.249213, + "nauc_recall_at_1000_diff1": 0.124005, + "nauc_precision_at_1_max": 0.427805, + "nauc_precision_at_1_std": -0.282245, + "nauc_precision_at_1_diff1": 0.78418, + "nauc_precision_at_3_max": 0.101025, + "nauc_precision_at_3_std": 0.101099, + "nauc_precision_at_3_diff1": -0.129135, + "nauc_precision_at_5_max": 0.082147, + "nauc_precision_at_5_std": 0.186908, + "nauc_precision_at_5_diff1": -0.31387, + "nauc_precision_at_10_max": 0.092508, + "nauc_precision_at_10_std": 0.298096, + "nauc_precision_at_10_diff1": -0.380323, + "nauc_precision_at_20_max": 0.037044, + "nauc_precision_at_20_std": 0.298925, + "nauc_precision_at_20_diff1": -0.401419, + "nauc_precision_at_100_max": -0.086747, + "nauc_precision_at_100_std": 0.204891, + "nauc_precision_at_100_diff1": -0.392207, + "nauc_precision_at_1000_max": -0.238845, + "nauc_precision_at_1000_std": 0.017347, + "nauc_precision_at_1000_diff1": -0.388405, + "nauc_mrr_at_1_max": 0.427805, + "nauc_mrr_at_1_std": -0.282245, + "nauc_mrr_at_1_diff1": 0.78418, + "nauc_mrr_at_3_max": 0.471097, + "nauc_mrr_at_3_std": -0.231022, + "nauc_mrr_at_3_diff1": 0.749352, + "nauc_mrr_at_5_max": 0.475708, + "nauc_mrr_at_5_std": -0.224856, + "nauc_mrr_at_5_diff1": 0.752514, + "nauc_mrr_at_10_max": 0.468118, + "nauc_mrr_at_10_std": -0.238903, + "nauc_mrr_at_10_diff1": 0.760199, + "nauc_mrr_at_20_max": 0.464674, + "nauc_mrr_at_20_std": -0.237552, + "nauc_mrr_at_20_diff1": 0.759283, + "nauc_mrr_at_100_max": 0.465481, + "nauc_mrr_at_100_std": -0.236775, + "nauc_mrr_at_100_diff1": 0.759767, + "nauc_mrr_at_1000_max": 0.465481, + "nauc_mrr_at_1000_std": -0.236775, + "nauc_mrr_at_1000_diff1": 0.759767, + "hit_rate_at_1": 0.84043, + "hit_rate_at_3": 0.93085, + "hit_rate_at_5": 0.95745, + "hit_rate_at_10": 0.98404, + "hit_rate_at_20": 0.99468, + "hit_rate_at_100": 1.0, + "hit_rate_at_1000": 1.0, + "main_score": 0.57945, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ], + "real_chat_test": [ + { + "ndcg_at_1": 0.82759, + "ndcg_at_3": 0.70819, + "ndcg_at_5": 0.66655, + "ndcg_at_10": 0.61782, + "ndcg_at_20": 0.56998, + "ndcg_at_100": 0.5185, + "ndcg_at_1000": 0.59622, + "map_at_1": 0.06169, + "map_at_3": 0.11006, + "map_at_5": 0.13194, + "map_at_10": 0.16109, + "map_at_20": 0.18331, + "map_at_100": 0.23659, + "map_at_1000": 0.29396, + "recall_at_1": 0.06169, + "recall_at_3": 0.11713, + "recall_at_5": 0.14764, + "recall_at_10": 0.20622, + "recall_at_20": 0.25624, + "recall_at_100": 0.42537, + "recall_at_1000": 0.7415, + "accuracy": 0.06169, + "precision_at_1": 0.82759, + "precision_at_3": 0.66379, + "precision_at_5": 0.6, + "precision_at_10": 0.51034, + "precision_at_20": 0.41121, + "precision_at_100": 0.24022, + "precision_at_1000": 0.08268, + "mrr_at_1": 0.827586, + "mrr_at_3": 0.877155, + "mrr_at_5": 0.882328, + "mrr_at_10": 0.886732, + "mrr_at_20": 0.887001, + "mrr_at_100": 0.887189, + "mrr_at_1000": 0.887202, + "nauc_ndcg_at_1_max": 0.539796, + "nauc_ndcg_at_1_std": 0.393151, + "nauc_ndcg_at_1_diff1": 0.472994, + "nauc_ndcg_at_3_max": 0.271689, + "nauc_ndcg_at_3_std": 0.189318, + "nauc_ndcg_at_3_diff1": 0.08778, + "nauc_ndcg_at_5_max": 0.228885, + "nauc_ndcg_at_5_std": 0.244399, + "nauc_ndcg_at_5_diff1": 0.036533, + "nauc_ndcg_at_10_max": 0.262916, + "nauc_ndcg_at_10_std": 0.258765, + "nauc_ndcg_at_10_diff1": 0.05939, + "nauc_ndcg_at_20_max": 0.294168, + "nauc_ndcg_at_20_std": 0.262843, + "nauc_ndcg_at_20_diff1": 0.061037, + "nauc_ndcg_at_100_max": 0.366686, + "nauc_ndcg_at_100_std": 0.227206, + "nauc_ndcg_at_100_diff1": 0.163171, + "nauc_ndcg_at_1000_max": 0.339969, + "nauc_ndcg_at_1000_std": 0.346671, + "nauc_ndcg_at_1000_diff1": 0.196213, + "nauc_map_at_1_max": 0.373665, + "nauc_map_at_1_std": 0.02564, + "nauc_map_at_1_diff1": 0.54412, + "nauc_map_at_3_max": 0.309934, + "nauc_map_at_3_std": -0.052218, + "nauc_map_at_3_diff1": 0.399719, + "nauc_map_at_5_max": 0.312767, + "nauc_map_at_5_std": -0.024797, + "nauc_map_at_5_diff1": 0.378282, + "nauc_map_at_10_max": 0.31022, + "nauc_map_at_10_std": -0.027492, + "nauc_map_at_10_diff1": 0.345461, + "nauc_map_at_20_max": 0.313162, + "nauc_map_at_20_std": 0.002503, + "nauc_map_at_20_diff1": 0.319427, + "nauc_map_at_100_max": 0.333851, + "nauc_map_at_100_std": 0.109455, + "nauc_map_at_100_diff1": 0.284414, + "nauc_map_at_1000_max": 0.36617, + "nauc_map_at_1000_std": 0.216178, + "nauc_map_at_1000_diff1": 0.241894, + "nauc_recall_at_1_max": 0.373665, + "nauc_recall_at_1_std": 0.02564, + "nauc_recall_at_1_diff1": 0.54412, + "nauc_recall_at_3_max": 0.297481, + "nauc_recall_at_3_std": -0.059297, + "nauc_recall_at_3_diff1": 0.363719, + "nauc_recall_at_5_max": 0.298574, + "nauc_recall_at_5_std": -0.023674, + "nauc_recall_at_5_diff1": 0.340269, + "nauc_recall_at_10_max": 0.26715, + "nauc_recall_at_10_std": -0.062117, + "nauc_recall_at_10_diff1": 0.285742, + "nauc_recall_at_20_max": 0.271488, + "nauc_recall_at_20_std": -0.015986, + "nauc_recall_at_20_diff1": 0.236276, + "nauc_recall_at_100_max": 0.265888, + "nauc_recall_at_100_std": 0.083499, + "nauc_recall_at_100_diff1": 0.205099, + "nauc_recall_at_1000_max": 0.237536, + "nauc_recall_at_1000_std": 0.245194, + "nauc_recall_at_1000_diff1": 0.159149, + "nauc_precision_at_1_max": 0.539796, + "nauc_precision_at_1_std": 0.393151, + "nauc_precision_at_1_diff1": 0.472994, + "nauc_precision_at_3_max": 0.173515, + "nauc_precision_at_3_std": 0.13559, + "nauc_precision_at_3_diff1": -0.035941, + "nauc_precision_at_5_max": 0.090586, + "nauc_precision_at_5_std": 0.213929, + "nauc_precision_at_5_diff1": -0.128902, + "nauc_precision_at_10_max": 0.066302, + "nauc_precision_at_10_std": 0.242132, + "nauc_precision_at_10_diff1": -0.173645, + "nauc_precision_at_20_max": 0.026302, + "nauc_precision_at_20_std": 0.261172, + "nauc_precision_at_20_diff1": -0.22162, + "nauc_precision_at_100_max": -0.000385, + "nauc_precision_at_100_std": 0.25173, + "nauc_precision_at_100_diff1": -0.222162, + "nauc_precision_at_1000_max": -0.013633, + "nauc_precision_at_1000_std": 0.09128, + "nauc_precision_at_1000_diff1": -0.208277, + "nauc_mrr_at_1_max": 0.539796, + "nauc_mrr_at_1_std": 0.393151, + "nauc_mrr_at_1_diff1": 0.472994, + "nauc_mrr_at_3_max": 0.497212, + "nauc_mrr_at_3_std": 0.40726, + "nauc_mrr_at_3_diff1": 0.438281, + "nauc_mrr_at_5_max": 0.503742, + "nauc_mrr_at_5_std": 0.417137, + "nauc_mrr_at_5_diff1": 0.443306, + "nauc_mrr_at_10_max": 0.510548, + "nauc_mrr_at_10_std": 0.415111, + "nauc_mrr_at_10_diff1": 0.449687, + "nauc_mrr_at_20_max": 0.510928, + "nauc_mrr_at_20_std": 0.41398, + "nauc_mrr_at_20_diff1": 0.449427, + "nauc_mrr_at_100_max": 0.51008, + "nauc_mrr_at_100_std": 0.412964, + "nauc_mrr_at_100_diff1": 0.448953, + "nauc_mrr_at_1000_max": 0.510098, + "nauc_mrr_at_1000_std": 0.412947, + "nauc_mrr_at_1000_diff1": 0.448903, + "hit_rate_at_1": 0.82759, + "hit_rate_at_3": 0.93534, + "hit_rate_at_5": 0.9569, + "hit_rate_at_10": 0.98707, + "hit_rate_at_20": 0.99138, + "hit_rate_at_100": 0.99569, + "hit_rate_at_1000": 1.0, + "main_score": 0.61782, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 201.11935949325562, + "kg_co2_emissions": null, + "date": 1777885896.115211 +} \ No newline at end of file diff --git a/results/microsoft__harrier-oss-v1-0.6b/f9b9dc8d367d443f2479d27aa5d8d2850c0774ee/GlobalNewsRetrieval.json b/results/microsoft__harrier-oss-v1-0.6b/f9b9dc8d367d443f2479d27aa5d8d2850c0774ee/GlobalNewsRetrieval.json new file mode 100644 index 0000000000..cb3672e903 --- /dev/null +++ b/results/microsoft__harrier-oss-v1-0.6b/f9b9dc8d367d443f2479d27aa5d8d2850c0774ee/GlobalNewsRetrieval.json @@ -0,0 +1,1562 @@ +{ + "dataset_revision": "234dcd32922d32c78e08bfe506a2b90a5401cf7c", + "task_name": "GlobalNewsRetrieval", + "mteb_version": "2.12.38", + "scores": { + "test": [ + { + "ndcg_at_1": 0.71554, + "ndcg_at_3": 0.77762, + "ndcg_at_5": 0.80727, + "ndcg_at_10": 0.82207, + "ndcg_at_20": 0.82839, + "ndcg_at_100": 0.83711, + "ndcg_at_1000": 0.83764, + "map_at_1": 0.68087, + "map_at_3": 0.75359, + "map_at_5": 0.77205, + "map_at_10": 0.77961, + "map_at_20": 0.78172, + "map_at_100": 0.78331, + "map_at_1000": 0.78335, + "recall_at_1": 0.68087, + "recall_at_3": 0.82289, + "recall_at_5": 0.89098, + "recall_at_10": 0.93212, + "recall_at_20": 0.9551, + "recall_at_100": 0.99645, + "recall_at_1000": 1.0, + "accuracy": 0.68087, + "precision_at_1": 0.71554, + "precision_at_3": 0.29866, + "precision_at_5": 0.19624, + "precision_at_10": 0.10376, + "precision_at_20": 0.05357, + "precision_at_100": 0.01127, + "precision_at_1000": 0.00113, + "mrr_at_1": 0.715539, + "mrr_at_3": 0.777987, + "mrr_at_5": 0.792272, + "mrr_at_10": 0.796935, + "mrr_at_20": 0.79817, + "mrr_at_100": 0.799269, + "mrr_at_1000": 0.799286, + "nauc_ndcg_at_1_max": 0.478285, + "nauc_ndcg_at_1_std": -0.325308, + "nauc_ndcg_at_1_diff1": 0.767967, + "nauc_ndcg_at_3_max": 0.491933, + "nauc_ndcg_at_3_std": -0.356143, + "nauc_ndcg_at_3_diff1": 0.754489, + "nauc_ndcg_at_5_max": 0.505301, + "nauc_ndcg_at_5_std": -0.338503, + "nauc_ndcg_at_5_diff1": 0.741646, + "nauc_ndcg_at_10_max": 0.5089, + "nauc_ndcg_at_10_std": -0.323885, + "nauc_ndcg_at_10_diff1": 0.742358, + "nauc_ndcg_at_20_max": 0.507465, + "nauc_ndcg_at_20_std": -0.323132, + "nauc_ndcg_at_20_diff1": 0.747143, + "nauc_ndcg_at_100_max": 0.502403, + "nauc_ndcg_at_100_std": -0.327422, + "nauc_ndcg_at_100_diff1": 0.750112, + "nauc_ndcg_at_1000_max": 0.501242, + "nauc_ndcg_at_1000_std": -0.330066, + "nauc_ndcg_at_1000_diff1": 0.750504, + "nauc_map_at_1_max": 0.417371, + "nauc_map_at_1_std": -0.310704, + "nauc_map_at_1_diff1": 0.761814, + "nauc_map_at_3_max": 0.467305, + "nauc_map_at_3_std": -0.347587, + "nauc_map_at_3_diff1": 0.756174, + "nauc_map_at_5_max": 0.478908, + "nauc_map_at_5_std": -0.342919, + "nauc_map_at_5_diff1": 0.750305, + "nauc_map_at_10_max": 0.482517, + "nauc_map_at_10_std": -0.337161, + "nauc_map_at_10_diff1": 0.749705, + "nauc_map_at_20_max": 0.482025, + "nauc_map_at_20_std": -0.33679, + "nauc_map_at_20_diff1": 0.750929, + "nauc_map_at_100_max": 0.481435, + "nauc_map_at_100_std": -0.337535, + "nauc_map_at_100_diff1": 0.751601, + "nauc_map_at_1000_max": 0.481383, + "nauc_map_at_1000_std": -0.337646, + "nauc_map_at_1000_diff1": 0.75161, + "nauc_recall_at_1_max": 0.417371, + "nauc_recall_at_1_std": -0.310704, + "nauc_recall_at_1_diff1": 0.761814, + "nauc_recall_at_3_max": 0.505054, + "nauc_recall_at_3_std": -0.381825, + "nauc_recall_at_3_diff1": 0.730829, + "nauc_recall_at_5_max": 0.549477, + "nauc_recall_at_5_std": -0.325844, + "nauc_recall_at_5_diff1": 0.677229, + "nauc_recall_at_10_max": 0.587487, + "nauc_recall_at_10_std": -0.221159, + "nauc_recall_at_10_diff1": 0.654942, + "nauc_recall_at_20_max": 0.61524, + "nauc_recall_at_20_std": -0.175351, + "nauc_recall_at_20_diff1": 0.681538, + "nauc_recall_at_100_max": 0.812515, + "nauc_recall_at_100_std": 0.451948, + "nauc_recall_at_100_diff1": 0.589765, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.478285, + "nauc_precision_at_1_std": -0.325308, + "nauc_precision_at_1_diff1": 0.767967, + "nauc_precision_at_3_max": 0.435727, + "nauc_precision_at_3_std": -0.242733, + "nauc_precision_at_3_diff1": 0.400648, + "nauc_precision_at_5_max": 0.371575, + "nauc_precision_at_5_std": -0.114508, + "nauc_precision_at_5_diff1": 0.145318, + "nauc_precision_at_10_max": 0.258028, + "nauc_precision_at_10_std": 0.017802, + "nauc_precision_at_10_diff1": -0.047327, + "nauc_precision_at_20_max": 0.173277, + "nauc_precision_at_20_std": 0.066225, + "nauc_precision_at_20_diff1": -0.134583, + "nauc_precision_at_100_max": 0.048058, + "nauc_precision_at_100_std": 0.158273, + "nauc_precision_at_100_diff1": -0.296776, + "nauc_precision_at_1000_max": 0.032026, + "nauc_precision_at_1000_std": 0.150676, + "nauc_precision_at_1000_diff1": -0.307814, + "nauc_mrr_at_1_max": 0.478285, + "nauc_mrr_at_1_std": -0.325308, + "nauc_mrr_at_1_diff1": 0.767967, + "nauc_mrr_at_3_max": 0.519766, + "nauc_mrr_at_3_std": -0.347887, + "nauc_mrr_at_3_diff1": 0.757113, + "nauc_mrr_at_5_max": 0.520825, + "nauc_mrr_at_5_std": -0.330066, + "nauc_mrr_at_5_diff1": 0.750494, + "nauc_mrr_at_10_max": 0.517728, + "nauc_mrr_at_10_std": -0.327101, + "nauc_mrr_at_10_diff1": 0.753248, + "nauc_mrr_at_20_max": 0.517374, + "nauc_mrr_at_20_std": -0.327494, + "nauc_mrr_at_20_diff1": 0.754486, + "nauc_mrr_at_100_max": 0.516101, + "nauc_mrr_at_100_std": -0.329119, + "nauc_mrr_at_100_diff1": 0.754707, + "nauc_mrr_at_1000_max": 0.516056, + "nauc_mrr_at_1000_std": -0.329213, + "nauc_mrr_at_1000_diff1": 0.754705, + "hit_rate_at_1": 0.71554, + "hit_rate_at_3": 0.85088, + "hit_rate_at_5": 0.91353, + "hit_rate_at_10": 0.94612, + "hit_rate_at_20": 0.96491, + "hit_rate_at_100": 0.99749, + "hit_rate_at_1000": 1.0, + "main_score": 0.82207, + "hf_subset": "ar", + "languages": [ + "ara-Arab" + ] + }, + { + "ndcg_at_1": 0.79892, + "ndcg_at_3": 0.83938, + "ndcg_at_5": 0.85673, + "ndcg_at_10": 0.86401, + "ndcg_at_20": 0.87004, + "ndcg_at_100": 0.87784, + "ndcg_at_1000": 0.87942, + "map_at_1": 0.74023, + "map_at_3": 0.81903, + "map_at_5": 0.83059, + "map_at_10": 0.83487, + "map_at_20": 0.83686, + "map_at_100": 0.83813, + "map_at_1000": 0.83824, + "recall_at_1": 0.74023, + "recall_at_3": 0.87142, + "recall_at_5": 0.9103, + "recall_at_10": 0.92984, + "recall_at_20": 0.95108, + "recall_at_100": 0.98961, + "recall_at_1000": 1.0, + "accuracy": 0.74023, + "precision_at_1": 0.79892, + "precision_at_3": 0.33118, + "precision_at_5": 0.20968, + "precision_at_10": 0.10817, + "precision_at_20": 0.05575, + "precision_at_100": 0.01169, + "precision_at_1000": 0.00118, + "mrr_at_1": 0.798925, + "mrr_at_3": 0.845699, + "mrr_at_5": 0.852849, + "mrr_at_10": 0.854369, + "mrr_at_20": 0.855624, + "mrr_at_100": 0.856481, + "mrr_at_1000": 0.856538, + "nauc_ndcg_at_1_max": 0.527111, + "nauc_ndcg_at_1_std": -0.590093, + "nauc_ndcg_at_1_diff1": 0.81875, + "nauc_ndcg_at_3_max": 0.543516, + "nauc_ndcg_at_3_std": -0.594382, + "nauc_ndcg_at_3_diff1": 0.771022, + "nauc_ndcg_at_5_max": 0.56637, + "nauc_ndcg_at_5_std": -0.634817, + "nauc_ndcg_at_5_diff1": 0.777407, + "nauc_ndcg_at_10_max": 0.56706, + "nauc_ndcg_at_10_std": -0.628922, + "nauc_ndcg_at_10_diff1": 0.781457, + "nauc_ndcg_at_20_max": 0.558442, + "nauc_ndcg_at_20_std": -0.618885, + "nauc_ndcg_at_20_diff1": 0.782917, + "nauc_ndcg_at_100_max": 0.553596, + "nauc_ndcg_at_100_std": -0.601252, + "nauc_ndcg_at_100_diff1": 0.782422, + "nauc_ndcg_at_1000_max": 0.552332, + "nauc_ndcg_at_1000_std": -0.6058, + "nauc_ndcg_at_1000_diff1": 0.784666, + "nauc_map_at_1_max": 0.397066, + "nauc_map_at_1_std": -0.524094, + "nauc_map_at_1_diff1": 0.793466, + "nauc_map_at_3_max": 0.519165, + "nauc_map_at_3_std": -0.578787, + "nauc_map_at_3_diff1": 0.775264, + "nauc_map_at_5_max": 0.537262, + "nauc_map_at_5_std": -0.602295, + "nauc_map_at_5_diff1": 0.779837, + "nauc_map_at_10_max": 0.538395, + "nauc_map_at_10_std": -0.600841, + "nauc_map_at_10_diff1": 0.781712, + "nauc_map_at_20_max": 0.536255, + "nauc_map_at_20_std": -0.598471, + "nauc_map_at_20_diff1": 0.782385, + "nauc_map_at_100_max": 0.535587, + "nauc_map_at_100_std": -0.595684, + "nauc_map_at_100_diff1": 0.782323, + "nauc_map_at_1000_max": 0.535575, + "nauc_map_at_1000_std": -0.595824, + "nauc_map_at_1000_diff1": 0.78241, + "nauc_recall_at_1_max": 0.397066, + "nauc_recall_at_1_std": -0.524094, + "nauc_recall_at_1_diff1": 0.793466, + "nauc_recall_at_3_max": 0.568722, + "nauc_recall_at_3_std": -0.60342, + "nauc_recall_at_3_diff1": 0.721481, + "nauc_recall_at_5_max": 0.635552, + "nauc_recall_at_5_std": -0.737685, + "nauc_recall_at_5_diff1": 0.714744, + "nauc_recall_at_10_max": 0.660296, + "nauc_recall_at_10_std": -0.733473, + "nauc_recall_at_10_diff1": 0.729193, + "nauc_recall_at_20_max": 0.634967, + "nauc_recall_at_20_std": -0.700737, + "nauc_recall_at_20_diff1": 0.736407, + "nauc_recall_at_100_max": 0.659189, + "nauc_recall_at_100_std": -0.204224, + "nauc_recall_at_100_diff1": 0.58974, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.527111, + "nauc_precision_at_1_std": -0.590093, + "nauc_precision_at_1_diff1": 0.81875, + "nauc_precision_at_3_max": 0.337812, + "nauc_precision_at_3_std": -0.1682, + "nauc_precision_at_3_diff1": 0.087587, + "nauc_precision_at_5_max": 0.277858, + "nauc_precision_at_5_std": -0.109342, + "nauc_precision_at_5_diff1": -0.046374, + "nauc_precision_at_10_max": 0.20539, + "nauc_precision_at_10_std": -0.023805, + "nauc_precision_at_10_diff1": -0.116243, + "nauc_precision_at_20_max": 0.12436, + "nauc_precision_at_20_std": 0.058323, + "nauc_precision_at_20_diff1": -0.190146, + "nauc_precision_at_100_max": 0.003379, + "nauc_precision_at_100_std": 0.220503, + "nauc_precision_at_100_diff1": -0.338133, + "nauc_precision_at_1000_max": -0.024012, + "nauc_precision_at_1000_std": 0.237842, + "nauc_precision_at_1000_diff1": -0.366136, + "nauc_mrr_at_1_max": 0.527111, + "nauc_mrr_at_1_std": -0.590093, + "nauc_mrr_at_1_diff1": 0.81875, + "nauc_mrr_at_3_max": 0.564842, + "nauc_mrr_at_3_std": -0.616487, + "nauc_mrr_at_3_diff1": 0.796047, + "nauc_mrr_at_5_max": 0.564785, + "nauc_mrr_at_5_std": -0.634093, + "nauc_mrr_at_5_diff1": 0.799306, + "nauc_mrr_at_10_max": 0.563951, + "nauc_mrr_at_10_std": -0.631183, + "nauc_mrr_at_10_diff1": 0.800346, + "nauc_mrr_at_20_max": 0.561733, + "nauc_mrr_at_20_std": -0.62727, + "nauc_mrr_at_20_diff1": 0.799989, + "nauc_mrr_at_100_max": 0.561807, + "nauc_mrr_at_100_std": -0.625279, + "nauc_mrr_at_100_diff1": 0.800002, + "nauc_mrr_at_1000_max": 0.5617, + "nauc_mrr_at_1000_std": -0.625498, + "nauc_mrr_at_1000_diff1": 0.800106, + "hit_rate_at_1": 0.79892, + "hit_rate_at_3": 0.90215, + "hit_rate_at_5": 0.93333, + "hit_rate_at_10": 0.94409, + "hit_rate_at_20": 0.96129, + "hit_rate_at_100": 0.99247, + "hit_rate_at_1000": 1.0, + "main_score": 0.86401, + "hf_subset": "zh", + "languages": [ + "zho-Hans" + ] + }, + { + "ndcg_at_1": 0.7369, + "ndcg_at_3": 0.80246, + "ndcg_at_5": 0.82088, + "ndcg_at_10": 0.83708, + "ndcg_at_20": 0.84613, + "ndcg_at_100": 0.85075, + "ndcg_at_1000": 0.8513, + "map_at_1": 0.70949, + "map_at_3": 0.7802, + "map_at_5": 0.79103, + "map_at_10": 0.79864, + "map_at_20": 0.80149, + "map_at_100": 0.80241, + "map_at_1000": 0.80245, + "recall_at_1": 0.70949, + "recall_at_3": 0.85166, + "recall_at_5": 0.89457, + "recall_at_10": 0.94065, + "recall_at_20": 0.97448, + "recall_at_100": 0.9964, + "recall_at_1000": 1.0, + "accuracy": 0.70949, + "precision_at_1": 0.7369, + "precision_at_3": 0.30079, + "precision_at_5": 0.19075, + "precision_at_10": 0.10164, + "precision_at_20": 0.05303, + "precision_at_100": 0.01088, + "precision_at_1000": 0.00109, + "mrr_at_1": 0.736896, + "mrr_at_3": 0.796506, + "mrr_at_5": 0.806115, + "mrr_at_10": 0.811829, + "mrr_at_20": 0.813639, + "mrr_at_100": 0.814219, + "mrr_at_1000": 0.814235, + "nauc_ndcg_at_1_max": 0.46248, + "nauc_ndcg_at_1_std": -0.473293, + "nauc_ndcg_at_1_diff1": 0.827513, + "nauc_ndcg_at_3_max": 0.498023, + "nauc_ndcg_at_3_std": -0.564922, + "nauc_ndcg_at_3_diff1": 0.797985, + "nauc_ndcg_at_5_max": 0.494725, + "nauc_ndcg_at_5_std": -0.577876, + "nauc_ndcg_at_5_diff1": 0.796818, + "nauc_ndcg_at_10_max": 0.498774, + "nauc_ndcg_at_10_std": -0.567371, + "nauc_ndcg_at_10_diff1": 0.803566, + "nauc_ndcg_at_20_max": 0.499845, + "nauc_ndcg_at_20_std": -0.546717, + "nauc_ndcg_at_20_diff1": 0.804151, + "nauc_ndcg_at_100_max": 0.493179, + "nauc_ndcg_at_100_std": -0.543737, + "nauc_ndcg_at_100_diff1": 0.807291, + "nauc_ndcg_at_1000_max": 0.492388, + "nauc_ndcg_at_1000_std": -0.540647, + "nauc_ndcg_at_1000_diff1": 0.807268, + "nauc_map_at_1_max": 0.415952, + "nauc_map_at_1_std": -0.484698, + "nauc_map_at_1_diff1": 0.818228, + "nauc_map_at_3_max": 0.479448, + "nauc_map_at_3_std": -0.545632, + "nauc_map_at_3_diff1": 0.8003, + "nauc_map_at_5_max": 0.478768, + "nauc_map_at_5_std": -0.552215, + "nauc_map_at_5_diff1": 0.800014, + "nauc_map_at_10_max": 0.481778, + "nauc_map_at_10_std": -0.547451, + "nauc_map_at_10_diff1": 0.8033, + "nauc_map_at_20_max": 0.482098, + "nauc_map_at_20_std": -0.541535, + "nauc_map_at_20_diff1": 0.80356, + "nauc_map_at_100_max": 0.481133, + "nauc_map_at_100_std": -0.54103, + "nauc_map_at_100_diff1": 0.8042, + "nauc_map_at_1000_max": 0.481083, + "nauc_map_at_1000_std": -0.540829, + "nauc_map_at_1000_diff1": 0.804189, + "nauc_recall_at_1_max": 0.415952, + "nauc_recall_at_1_std": -0.484698, + "nauc_recall_at_1_diff1": 0.818228, + "nauc_recall_at_3_max": 0.534049, + "nauc_recall_at_3_std": -0.656267, + "nauc_recall_at_3_diff1": 0.764991, + "nauc_recall_at_5_max": 0.527254, + "nauc_recall_at_5_std": -0.737364, + "nauc_recall_at_5_diff1": 0.748371, + "nauc_recall_at_10_max": 0.567201, + "nauc_recall_at_10_std": -0.790954, + "nauc_recall_at_10_diff1": 0.75409, + "nauc_recall_at_20_max": 0.685861, + "nauc_recall_at_20_std": -0.706438, + "nauc_recall_at_20_diff1": 0.718788, + "nauc_recall_at_100_max": 0.661195, + "nauc_recall_at_100_std": -1.20773, + "nauc_recall_at_100_diff1": 0.771564, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.46248, + "nauc_precision_at_1_std": -0.473293, + "nauc_precision_at_1_diff1": 0.827513, + "nauc_precision_at_3_max": 0.464792, + "nauc_precision_at_3_std": -0.416254, + "nauc_precision_at_3_diff1": 0.446485, + "nauc_precision_at_5_max": 0.385124, + "nauc_precision_at_5_std": -0.357228, + "nauc_precision_at_5_diff1": 0.280784, + "nauc_precision_at_10_max": 0.256974, + "nauc_precision_at_10_std": -0.076751, + "nauc_precision_at_10_diff1": 0.022424, + "nauc_precision_at_20_max": 0.131423, + "nauc_precision_at_20_std": 0.168721, + "nauc_precision_at_20_diff1": -0.181532, + "nauc_precision_at_100_max": 0.015469, + "nauc_precision_at_100_std": 0.277211, + "nauc_precision_at_100_diff1": -0.296201, + "nauc_precision_at_1000_max": -0.005083, + "nauc_precision_at_1000_std": 0.312885, + "nauc_precision_at_1000_diff1": -0.314467, + "nauc_mrr_at_1_max": 0.46248, + "nauc_mrr_at_1_std": -0.473293, + "nauc_mrr_at_1_diff1": 0.827513, + "nauc_mrr_at_3_max": 0.505074, + "nauc_mrr_at_3_std": -0.526275, + "nauc_mrr_at_3_diff1": 0.814015, + "nauc_mrr_at_5_max": 0.502321, + "nauc_mrr_at_5_std": -0.52804, + "nauc_mrr_at_5_diff1": 0.81396, + "nauc_mrr_at_10_max": 0.498993, + "nauc_mrr_at_10_std": -0.52377, + "nauc_mrr_at_10_diff1": 0.815648, + "nauc_mrr_at_20_max": 0.498749, + "nauc_mrr_at_20_std": -0.520104, + "nauc_mrr_at_20_diff1": 0.815784, + "nauc_mrr_at_100_max": 0.498005, + "nauc_mrr_at_100_std": -0.519857, + "nauc_mrr_at_100_diff1": 0.816027, + "nauc_mrr_at_1000_max": 0.49798, + "nauc_mrr_at_1000_std": -0.519794, + "nauc_mrr_at_1000_diff1": 0.816024, + "hit_rate_at_1": 0.7369, + "hit_rate_at_3": 0.8705, + "hit_rate_at_5": 0.91264, + "hit_rate_at_10": 0.95375, + "hit_rate_at_20": 0.98047, + "hit_rate_at_100": 0.99794, + "hit_rate_at_1000": 1.0, + "main_score": 0.83708, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.75894, + "ndcg_at_3": 0.83526, + "ndcg_at_5": 0.85079, + "ndcg_at_10": 0.86302, + "ndcg_at_20": 0.86786, + "ndcg_at_100": 0.87211, + "ndcg_at_1000": 0.87211, + "map_at_1": 0.73493, + "map_at_3": 0.81299, + "map_at_5": 0.82204, + "map_at_10": 0.828, + "map_at_20": 0.82947, + "map_at_100": 0.83015, + "map_at_1000": 0.83015, + "recall_at_1": 0.73493, + "recall_at_3": 0.88832, + "recall_at_5": 0.92475, + "recall_at_10": 0.95982, + "recall_at_20": 0.97821, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "accuracy": 0.73493, + "precision_at_1": 0.75894, + "precision_at_3": 0.31222, + "precision_at_5": 0.19612, + "precision_at_10": 0.10245, + "precision_at_20": 0.05235, + "precision_at_100": 0.01073, + "precision_at_1000": 0.00107, + "mrr_at_1": 0.758938, + "mrr_at_3": 0.82414, + "mrr_at_5": 0.832159, + "mrr_at_10": 0.836334, + "mrr_at_20": 0.837399, + "mrr_at_100": 0.837896, + "mrr_at_1000": 0.837896, + "nauc_ndcg_at_1_max": 0.494828, + "nauc_ndcg_at_1_std": -0.410769, + "nauc_ndcg_at_1_diff1": 0.774614, + "nauc_ndcg_at_3_max": 0.532214, + "nauc_ndcg_at_3_std": -0.487873, + "nauc_ndcg_at_3_diff1": 0.745805, + "nauc_ndcg_at_5_max": 0.542874, + "nauc_ndcg_at_5_std": -0.480356, + "nauc_ndcg_at_5_diff1": 0.745079, + "nauc_ndcg_at_10_max": 0.542556, + "nauc_ndcg_at_10_std": -0.469543, + "nauc_ndcg_at_10_diff1": 0.747905, + "nauc_ndcg_at_20_max": 0.536109, + "nauc_ndcg_at_20_std": -0.464807, + "nauc_ndcg_at_20_diff1": 0.753245, + "nauc_ndcg_at_100_max": 0.528563, + "nauc_ndcg_at_100_std": -0.455078, + "nauc_ndcg_at_100_diff1": 0.753257, + "nauc_ndcg_at_1000_max": 0.528563, + "nauc_ndcg_at_1000_std": -0.455078, + "nauc_ndcg_at_1000_diff1": 0.753257, + "nauc_map_at_1_max": 0.462713, + "nauc_map_at_1_std": -0.403221, + "nauc_map_at_1_diff1": 0.761546, + "nauc_map_at_3_max": 0.516854, + "nauc_map_at_3_std": -0.46693, + "nauc_map_at_3_diff1": 0.749945, + "nauc_map_at_5_max": 0.522633, + "nauc_map_at_5_std": -0.462499, + "nauc_map_at_5_diff1": 0.749554, + "nauc_map_at_10_max": 0.522798, + "nauc_map_at_10_std": -0.458036, + "nauc_map_at_10_diff1": 0.750492, + "nauc_map_at_20_max": 0.52105, + "nauc_map_at_20_std": -0.456449, + "nauc_map_at_20_diff1": 0.751983, + "nauc_map_at_100_max": 0.520299, + "nauc_map_at_100_std": -0.454698, + "nauc_map_at_100_diff1": 0.751889, + "nauc_map_at_1000_max": 0.520299, + "nauc_map_at_1000_std": -0.454698, + "nauc_map_at_1000_diff1": 0.751889, + "nauc_recall_at_1_max": 0.462713, + "nauc_recall_at_1_std": -0.403221, + "nauc_recall_at_1_diff1": 0.761546, + "nauc_recall_at_3_max": 0.58225, + "nauc_recall_at_3_std": -0.593306, + "nauc_recall_at_3_diff1": 0.709072, + "nauc_recall_at_5_max": 0.643265, + "nauc_recall_at_5_std": -0.598896, + "nauc_recall_at_5_diff1": 0.683725, + "nauc_recall_at_10_max": 0.726153, + "nauc_recall_at_10_std": -0.619022, + "nauc_recall_at_10_diff1": 0.673425, + "nauc_recall_at_20_max": 0.75313, + "nauc_recall_at_20_std": -0.66703, + "nauc_recall_at_20_diff1": 0.728626, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.494828, + "nauc_precision_at_1_std": -0.410769, + "nauc_precision_at_1_diff1": 0.774614, + "nauc_precision_at_3_max": 0.392297, + "nauc_precision_at_3_std": -0.330064, + "nauc_precision_at_3_diff1": 0.337243, + "nauc_precision_at_5_max": 0.331082, + "nauc_precision_at_5_std": -0.207982, + "nauc_precision_at_5_diff1": 0.159595, + "nauc_precision_at_10_max": 0.183201, + "nauc_precision_at_10_std": -0.026905, + "nauc_precision_at_10_diff1": -0.060096, + "nauc_precision_at_20_max": 0.069605, + "nauc_precision_at_20_std": 0.075786, + "nauc_precision_at_20_diff1": -0.15917, + "nauc_precision_at_100_max": -0.068664, + "nauc_precision_at_100_std": 0.220502, + "nauc_precision_at_100_diff1": -0.311721, + "nauc_precision_at_1000_max": -0.068664, + "nauc_precision_at_1000_std": 0.220502, + "nauc_precision_at_1000_diff1": -0.311721, + "nauc_mrr_at_1_max": 0.494828, + "nauc_mrr_at_1_std": -0.410769, + "nauc_mrr_at_1_diff1": 0.774614, + "nauc_mrr_at_3_max": 0.524681, + "nauc_mrr_at_3_std": -0.455211, + "nauc_mrr_at_3_diff1": 0.759389, + "nauc_mrr_at_5_max": 0.526601, + "nauc_mrr_at_5_std": -0.452256, + "nauc_mrr_at_5_diff1": 0.760274, + "nauc_mrr_at_10_max": 0.525466, + "nauc_mrr_at_10_std": -0.447497, + "nauc_mrr_at_10_diff1": 0.761691, + "nauc_mrr_at_20_max": 0.524249, + "nauc_mrr_at_20_std": -0.446792, + "nauc_mrr_at_20_diff1": 0.762374, + "nauc_mrr_at_100_max": 0.523409, + "nauc_mrr_at_100_std": -0.44553, + "nauc_mrr_at_100_diff1": 0.762326, + "nauc_mrr_at_1000_max": 0.523409, + "nauc_mrr_at_1000_std": -0.44553, + "nauc_mrr_at_1000_diff1": 0.762326, + "hit_rate_at_1": 0.75894, + "hit_rate_at_3": 0.90092, + "hit_rate_at_5": 0.93667, + "hit_rate_at_10": 0.96629, + "hit_rate_at_20": 0.98161, + "hit_rate_at_100": 1.0, + "hit_rate_at_1000": 1.0, + "main_score": 0.86302, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.75424, + "ndcg_at_3": 0.81853, + "ndcg_at_5": 0.82981, + "ndcg_at_10": 0.84251, + "ndcg_at_20": 0.84841, + "ndcg_at_100": 0.85532, + "ndcg_at_1000": 0.85708, + "map_at_1": 0.73076, + "map_at_3": 0.80001, + "map_at_5": 0.80651, + "map_at_10": 0.81195, + "map_at_20": 0.81387, + "map_at_100": 0.81493, + "map_at_1000": 0.81503, + "recall_at_1": 0.73076, + "recall_at_3": 0.8652, + "recall_at_5": 0.89239, + "recall_at_10": 0.9307, + "recall_at_20": 0.95224, + "recall_at_100": 0.98773, + "recall_at_1000": 1.0, + "accuracy": 0.73076, + "precision_at_1": 0.75424, + "precision_at_3": 0.30508, + "precision_at_5": 0.18919, + "precision_at_10": 0.09883, + "precision_at_20": 0.0509, + "precision_at_100": 0.01059, + "precision_at_1000": 0.00107, + "mrr_at_1": 0.754237, + "mrr_at_3": 0.806674, + "mrr_at_5": 0.812553, + "mrr_at_10": 0.81763, + "mrr_at_20": 0.818881, + "mrr_at_100": 0.819745, + "mrr_at_1000": 0.819832, + "nauc_ndcg_at_1_max": 0.483107, + "nauc_ndcg_at_1_std": -0.511075, + "nauc_ndcg_at_1_diff1": 0.769486, + "nauc_ndcg_at_3_max": 0.539099, + "nauc_ndcg_at_3_std": -0.586008, + "nauc_ndcg_at_3_diff1": 0.748242, + "nauc_ndcg_at_5_max": 0.543142, + "nauc_ndcg_at_5_std": -0.58693, + "nauc_ndcg_at_5_diff1": 0.746718, + "nauc_ndcg_at_10_max": 0.537161, + "nauc_ndcg_at_10_std": -0.598815, + "nauc_ndcg_at_10_diff1": 0.748656, + "nauc_ndcg_at_20_max": 0.533854, + "nauc_ndcg_at_20_std": -0.59905, + "nauc_ndcg_at_20_diff1": 0.747852, + "nauc_ndcg_at_100_max": 0.52842, + "nauc_ndcg_at_100_std": -0.585364, + "nauc_ndcg_at_100_diff1": 0.751466, + "nauc_ndcg_at_1000_max": 0.525789, + "nauc_ndcg_at_1000_std": -0.578916, + "nauc_ndcg_at_1000_diff1": 0.751388, + "nauc_map_at_1_max": 0.411734, + "nauc_map_at_1_std": -0.475576, + "nauc_map_at_1_diff1": 0.753846, + "nauc_map_at_3_max": 0.515063, + "nauc_map_at_3_std": -0.559641, + "nauc_map_at_3_diff1": 0.746776, + "nauc_map_at_5_max": 0.51687, + "nauc_map_at_5_std": -0.560522, + "nauc_map_at_5_diff1": 0.746819, + "nauc_map_at_10_max": 0.514699, + "nauc_map_at_10_std": -0.564007, + "nauc_map_at_10_diff1": 0.747185, + "nauc_map_at_20_max": 0.513516, + "nauc_map_at_20_std": -0.563868, + "nauc_map_at_20_diff1": 0.746925, + "nauc_map_at_100_max": 0.512747, + "nauc_map_at_100_std": -0.562386, + "nauc_map_at_100_diff1": 0.747629, + "nauc_map_at_1000_max": 0.512622, + "nauc_map_at_1000_std": -0.562107, + "nauc_map_at_1000_diff1": 0.747627, + "nauc_recall_at_1_max": 0.411734, + "nauc_recall_at_1_std": -0.475576, + "nauc_recall_at_1_diff1": 0.753846, + "nauc_recall_at_3_max": 0.606981, + "nauc_recall_at_3_std": -0.670455, + "nauc_recall_at_3_diff1": 0.73094, + "nauc_recall_at_5_max": 0.636013, + "nauc_recall_at_5_std": -0.692999, + "nauc_recall_at_5_diff1": 0.721755, + "nauc_recall_at_10_max": 0.642448, + "nauc_recall_at_10_std": -0.822819, + "nauc_recall_at_10_diff1": 0.721161, + "nauc_recall_at_20_max": 0.662013, + "nauc_recall_at_20_std": -0.928677, + "nauc_recall_at_20_diff1": 0.708243, + "nauc_recall_at_100_max": 0.722389, + "nauc_recall_at_100_std": -1.074258, + "nauc_recall_at_100_diff1": 0.755664, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.483107, + "nauc_precision_at_1_std": -0.511075, + "nauc_precision_at_1_diff1": 0.769486, + "nauc_precision_at_3_max": 0.484667, + "nauc_precision_at_3_std": -0.426145, + "nauc_precision_at_3_diff1": 0.358569, + "nauc_precision_at_5_max": 0.458567, + "nauc_precision_at_5_std": -0.382918, + "nauc_precision_at_5_diff1": 0.283641, + "nauc_precision_at_10_max": 0.386393, + "nauc_precision_at_10_std": -0.340092, + "nauc_precision_at_10_diff1": 0.141499, + "nauc_precision_at_20_max": 0.263592, + "nauc_precision_at_20_std": -0.2304, + "nauc_precision_at_20_diff1": -0.004305, + "nauc_precision_at_100_max": 0.114851, + "nauc_precision_at_100_std": 0.008595, + "nauc_precision_at_100_diff1": -0.212176, + "nauc_precision_at_1000_max": 0.042165, + "nauc_precision_at_1000_std": 0.117131, + "nauc_precision_at_1000_diff1": -0.307703, + "nauc_mrr_at_1_max": 0.483107, + "nauc_mrr_at_1_std": -0.511075, + "nauc_mrr_at_1_diff1": 0.769486, + "nauc_mrr_at_3_max": 0.534258, + "nauc_mrr_at_3_std": -0.572839, + "nauc_mrr_at_3_diff1": 0.763947, + "nauc_mrr_at_5_max": 0.535093, + "nauc_mrr_at_5_std": -0.570805, + "nauc_mrr_at_5_diff1": 0.761733, + "nauc_mrr_at_10_max": 0.531562, + "nauc_mrr_at_10_std": -0.574443, + "nauc_mrr_at_10_diff1": 0.762907, + "nauc_mrr_at_20_max": 0.530762, + "nauc_mrr_at_20_std": -0.573891, + "nauc_mrr_at_20_diff1": 0.762696, + "nauc_mrr_at_100_max": 0.529996, + "nauc_mrr_at_100_std": -0.572573, + "nauc_mrr_at_100_diff1": 0.763293, + "nauc_mrr_at_1000_max": 0.529908, + "nauc_mrr_at_1000_std": -0.57227, + "nauc_mrr_at_1000_diff1": 0.763302, + "hit_rate_at_1": 0.75424, + "hit_rate_at_3": 0.87288, + "hit_rate_at_5": 0.89936, + "hit_rate_at_10": 0.9375, + "hit_rate_at_20": 0.95551, + "hit_rate_at_100": 0.98835, + "hit_rate_at_1000": 1.0, + "main_score": 0.84251, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ] + }, + { + "ndcg_at_1": 0.64174, + "ndcg_at_3": 0.70812, + "ndcg_at_5": 0.72699, + "ndcg_at_10": 0.74035, + "ndcg_at_20": 0.75177, + "ndcg_at_100": 0.76782, + "ndcg_at_1000": 0.77233, + "map_at_1": 0.62115, + "map_at_3": 0.68709, + "map_at_5": 0.698, + "map_at_10": 0.70366, + "map_at_20": 0.70686, + "map_at_100": 0.70934, + "map_at_1000": 0.70955, + "recall_at_1": 0.62115, + "recall_at_3": 0.75649, + "recall_at_5": 0.8008, + "recall_at_10": 0.84095, + "recall_at_20": 0.88422, + "recall_at_100": 0.96729, + "recall_at_1000": 1.0, + "accuracy": 0.62115, + "precision_at_1": 0.64174, + "precision_at_3": 0.26376, + "precision_at_5": 0.16802, + "precision_at_10": 0.08858, + "precision_at_20": 0.04688, + "precision_at_100": 0.01037, + "precision_at_1000": 0.00108, + "mrr_at_1": 0.641745, + "mrr_at_3": 0.700415, + "mrr_at_5": 0.710592, + "mrr_at_10": 0.715983, + "mrr_at_20": 0.719214, + "mrr_at_100": 0.721011, + "mrr_at_1000": 0.721153, + "nauc_ndcg_at_1_max": 0.556357, + "nauc_ndcg_at_1_std": -0.43168, + "nauc_ndcg_at_1_diff1": 0.777373, + "nauc_ndcg_at_3_max": 0.582372, + "nauc_ndcg_at_3_std": -0.442969, + "nauc_ndcg_at_3_diff1": 0.729832, + "nauc_ndcg_at_5_max": 0.575673, + "nauc_ndcg_at_5_std": -0.456189, + "nauc_ndcg_at_5_diff1": 0.720202, + "nauc_ndcg_at_10_max": 0.581284, + "nauc_ndcg_at_10_std": -0.45089, + "nauc_ndcg_at_10_diff1": 0.724412, + "nauc_ndcg_at_20_max": 0.582655, + "nauc_ndcg_at_20_std": -0.454629, + "nauc_ndcg_at_20_diff1": 0.727784, + "nauc_ndcg_at_100_max": 0.577074, + "nauc_ndcg_at_100_std": -0.45323, + "nauc_ndcg_at_100_diff1": 0.732821, + "nauc_ndcg_at_1000_max": 0.576642, + "nauc_ndcg_at_1000_std": -0.449635, + "nauc_ndcg_at_1000_diff1": 0.734515, + "nauc_map_at_1_max": 0.528208, + "nauc_map_at_1_std": -0.406091, + "nauc_map_at_1_diff1": 0.757117, + "nauc_map_at_3_max": 0.568955, + "nauc_map_at_3_std": -0.435229, + "nauc_map_at_3_diff1": 0.732822, + "nauc_map_at_5_max": 0.56506, + "nauc_map_at_5_std": -0.442773, + "nauc_map_at_5_diff1": 0.727536, + "nauc_map_at_10_max": 0.567515, + "nauc_map_at_10_std": -0.440999, + "nauc_map_at_10_diff1": 0.730023, + "nauc_map_at_20_max": 0.568306, + "nauc_map_at_20_std": -0.442227, + "nauc_map_at_20_diff1": 0.73107, + "nauc_map_at_100_max": 0.567744, + "nauc_map_at_100_std": -0.441582, + "nauc_map_at_100_diff1": 0.731794, + "nauc_map_at_1000_max": 0.567771, + "nauc_map_at_1000_std": -0.441511, + "nauc_map_at_1000_diff1": 0.731901, + "nauc_recall_at_1_max": 0.528208, + "nauc_recall_at_1_std": -0.406091, + "nauc_recall_at_1_diff1": 0.757117, + "nauc_recall_at_3_max": 0.611762, + "nauc_recall_at_3_std": -0.448337, + "nauc_recall_at_3_diff1": 0.692457, + "nauc_recall_at_5_max": 0.594258, + "nauc_recall_at_5_std": -0.485482, + "nauc_recall_at_5_diff1": 0.655129, + "nauc_recall_at_10_max": 0.619246, + "nauc_recall_at_10_std": -0.464998, + "nauc_recall_at_10_diff1": 0.6538, + "nauc_recall_at_20_max": 0.638993, + "nauc_recall_at_20_std": -0.498142, + "nauc_recall_at_20_diff1": 0.656736, + "nauc_recall_at_100_max": 0.602953, + "nauc_recall_at_100_std": -0.609894, + "nauc_recall_at_100_diff1": 0.652445, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.556357, + "nauc_precision_at_1_std": -0.43168, + "nauc_precision_at_1_diff1": 0.777373, + "nauc_precision_at_3_max": 0.55706, + "nauc_precision_at_3_std": -0.42167, + "nauc_precision_at_3_diff1": 0.581532, + "nauc_precision_at_5_max": 0.506886, + "nauc_precision_at_5_std": -0.430108, + "nauc_precision_at_5_diff1": 0.498147, + "nauc_precision_at_10_max": 0.482118, + "nauc_precision_at_10_std": -0.370998, + "nauc_precision_at_10_diff1": 0.439572, + "nauc_precision_at_20_max": 0.42508, + "nauc_precision_at_20_std": -0.347827, + "nauc_precision_at_20_diff1": 0.351495, + "nauc_precision_at_100_max": 0.06587, + "nauc_precision_at_100_std": -0.081574, + "nauc_precision_at_100_diff1": -0.022062, + "nauc_precision_at_1000_max": -0.095309, + "nauc_precision_at_1000_std": 0.085079, + "nauc_precision_at_1000_diff1": -0.202072, + "nauc_mrr_at_1_max": 0.556357, + "nauc_mrr_at_1_std": -0.43168, + "nauc_mrr_at_1_diff1": 0.777373, + "nauc_mrr_at_3_max": 0.582375, + "nauc_mrr_at_3_std": -0.451759, + "nauc_mrr_at_3_diff1": 0.755683, + "nauc_mrr_at_5_max": 0.578266, + "nauc_mrr_at_5_std": -0.457807, + "nauc_mrr_at_5_diff1": 0.751584, + "nauc_mrr_at_10_max": 0.579862, + "nauc_mrr_at_10_std": -0.455768, + "nauc_mrr_at_10_diff1": 0.752795, + "nauc_mrr_at_20_max": 0.580061, + "nauc_mrr_at_20_std": -0.455757, + "nauc_mrr_at_20_diff1": 0.753528, + "nauc_mrr_at_100_max": 0.579358, + "nauc_mrr_at_100_std": -0.455724, + "nauc_mrr_at_100_diff1": 0.754143, + "nauc_mrr_at_1000_max": 0.57938, + "nauc_mrr_at_1000_std": -0.455563, + "nauc_mrr_at_1000_diff1": 0.754139, + "hit_rate_at_1": 0.64174, + "hit_rate_at_3": 0.77051, + "hit_rate_at_5": 0.81412, + "hit_rate_at_10": 0.85462, + "hit_rate_at_20": 0.90135, + "hit_rate_at_100": 0.973, + "hit_rate_at_1000": 1.0, + "main_score": 0.74035, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.834, + "ndcg_at_3": 0.88913, + "ndcg_at_5": 0.90134, + "ndcg_at_10": 0.91041, + "ndcg_at_20": 0.91326, + "ndcg_at_100": 0.91428, + "ndcg_at_1000": 0.91428, + "map_at_1": 0.81196, + "map_at_3": 0.87216, + "map_at_5": 0.8801, + "map_at_10": 0.88434, + "map_at_20": 0.8852, + "map_at_100": 0.88541, + "map_at_1000": 0.88541, + "recall_at_1": 0.81196, + "recall_at_3": 0.92916, + "recall_at_5": 0.95758, + "recall_at_10": 0.98466, + "recall_at_20": 0.99522, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "accuracy": 0.81196, + "precision_at_1": 0.834, + "precision_at_3": 0.32461, + "precision_at_5": 0.20201, + "precision_at_10": 0.10423, + "precision_at_20": 0.05272, + "precision_at_100": 0.0106, + "precision_at_1000": 0.00106, + "mrr_at_1": 0.834004, + "mrr_at_3": 0.880785, + "mrr_at_5": 0.886066, + "mrr_at_10": 0.889229, + "mrr_at_20": 0.890014, + "mrr_at_100": 0.89015, + "mrr_at_1000": 0.89015, + "nauc_ndcg_at_1_max": 0.547863, + "nauc_ndcg_at_1_std": -0.455612, + "nauc_ndcg_at_1_diff1": 0.841983, + "nauc_ndcg_at_3_max": 0.568996, + "nauc_ndcg_at_3_std": -0.545803, + "nauc_ndcg_at_3_diff1": 0.812057, + "nauc_ndcg_at_5_max": 0.573157, + "nauc_ndcg_at_5_std": -0.521303, + "nauc_ndcg_at_5_diff1": 0.815037, + "nauc_ndcg_at_10_max": 0.563444, + "nauc_ndcg_at_10_std": -0.512102, + "nauc_ndcg_at_10_diff1": 0.820197, + "nauc_ndcg_at_20_max": 0.564605, + "nauc_ndcg_at_20_std": -0.506379, + "nauc_ndcg_at_20_diff1": 0.823147, + "nauc_ndcg_at_100_max": 0.562435, + "nauc_ndcg_at_100_std": -0.502169, + "nauc_ndcg_at_100_diff1": 0.822217, + "nauc_ndcg_at_1000_max": 0.562435, + "nauc_ndcg_at_1000_std": -0.502169, + "nauc_ndcg_at_1000_diff1": 0.822217, + "nauc_map_at_1_max": 0.492314, + "nauc_map_at_1_std": -0.460657, + "nauc_map_at_1_diff1": 0.82813, + "nauc_map_at_3_max": 0.556864, + "nauc_map_at_3_std": -0.530961, + "nauc_map_at_3_diff1": 0.815909, + "nauc_map_at_5_max": 0.559362, + "nauc_map_at_5_std": -0.517694, + "nauc_map_at_5_diff1": 0.818563, + "nauc_map_at_10_max": 0.556836, + "nauc_map_at_10_std": -0.509805, + "nauc_map_at_10_diff1": 0.820624, + "nauc_map_at_20_max": 0.55699, + "nauc_map_at_20_std": -0.508851, + "nauc_map_at_20_diff1": 0.821321, + "nauc_map_at_100_max": 0.556553, + "nauc_map_at_100_std": -0.507904, + "nauc_map_at_100_diff1": 0.821187, + "nauc_map_at_1000_max": 0.556553, + "nauc_map_at_1000_std": -0.507904, + "nauc_map_at_1000_diff1": 0.821187, + "nauc_recall_at_1_max": 0.492314, + "nauc_recall_at_1_std": -0.460657, + "nauc_recall_at_1_diff1": 0.82813, + "nauc_recall_at_3_max": 0.597459, + "nauc_recall_at_3_std": -0.674991, + "nauc_recall_at_3_diff1": 0.771328, + "nauc_recall_at_5_max": 0.634661, + "nauc_recall_at_5_std": -0.619332, + "nauc_recall_at_5_diff1": 0.761219, + "nauc_recall_at_10_max": 0.578221, + "nauc_recall_at_10_std": -0.724421, + "nauc_recall_at_10_diff1": 0.773274, + "nauc_recall_at_20_max": 0.717716, + "nauc_recall_at_20_std": -0.790349, + "nauc_recall_at_20_diff1": 0.890556, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.547863, + "nauc_precision_at_1_std": -0.455612, + "nauc_precision_at_1_diff1": 0.841983, + "nauc_precision_at_3_max": 0.344243, + "nauc_precision_at_3_std": -0.234828, + "nauc_precision_at_3_diff1": 0.216036, + "nauc_precision_at_5_max": 0.207514, + "nauc_precision_at_5_std": -0.011751, + "nauc_precision_at_5_diff1": -0.00595, + "nauc_precision_at_10_max": 0.047876, + "nauc_precision_at_10_std": 0.189333, + "nauc_precision_at_10_diff1": -0.220088, + "nauc_precision_at_20_max": -0.005917, + "nauc_precision_at_20_std": 0.283714, + "nauc_precision_at_20_diff1": -0.314457, + "nauc_precision_at_100_max": -0.048262, + "nauc_precision_at_100_std": 0.339046, + "nauc_precision_at_100_diff1": -0.36525, + "nauc_precision_at_1000_max": -0.048262, + "nauc_precision_at_1000_std": 0.339046, + "nauc_precision_at_1000_diff1": -0.36525, + "nauc_mrr_at_1_max": 0.547863, + "nauc_mrr_at_1_std": -0.455612, + "nauc_mrr_at_1_diff1": 0.841983, + "nauc_mrr_at_3_max": 0.574662, + "nauc_mrr_at_3_std": -0.497705, + "nauc_mrr_at_3_diff1": 0.825933, + "nauc_mrr_at_5_max": 0.572383, + "nauc_mrr_at_5_std": -0.485254, + "nauc_mrr_at_5_diff1": 0.826392, + "nauc_mrr_at_10_max": 0.568429, + "nauc_mrr_at_10_std": -0.48587, + "nauc_mrr_at_10_diff1": 0.828078, + "nauc_mrr_at_20_max": 0.568349, + "nauc_mrr_at_20_std": -0.484798, + "nauc_mrr_at_20_diff1": 0.828441, + "nauc_mrr_at_100_max": 0.568107, + "nauc_mrr_at_100_std": -0.484479, + "nauc_mrr_at_100_diff1": 0.828314, + "nauc_mrr_at_1000_max": 0.568107, + "nauc_mrr_at_1000_std": -0.484479, + "nauc_mrr_at_1000_diff1": 0.828314, + "hit_rate_at_1": 0.834, + "hit_rate_at_3": 0.93863, + "hit_rate_at_5": 0.96177, + "hit_rate_at_10": 0.98592, + "hit_rate_at_20": 0.99598, + "hit_rate_at_100": 1.0, + "hit_rate_at_1000": 1.0, + "main_score": 0.91041, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ] + }, + { + "ndcg_at_1": 0.62955, + "ndcg_at_3": 0.70792, + "ndcg_at_5": 0.73157, + "ndcg_at_10": 0.75065, + "ndcg_at_20": 0.76054, + "ndcg_at_100": 0.77481, + "ndcg_at_1000": 0.77689, + "map_at_1": 0.60059, + "map_at_3": 0.68262, + "map_at_5": 0.69679, + "map_at_10": 0.70551, + "map_at_20": 0.70848, + "map_at_100": 0.71071, + "map_at_1000": 0.71082, + "recall_at_1": 0.60059, + "recall_at_3": 0.76465, + "recall_at_5": 0.81911, + "recall_at_10": 0.87426, + "recall_at_20": 0.91224, + "recall_at_100": 0.98526, + "recall_at_1000": 1.0, + "accuracy": 0.60059, + "precision_at_1": 0.62955, + "precision_at_3": 0.27471, + "precision_at_5": 0.17794, + "precision_at_10": 0.09563, + "precision_at_20": 0.05005, + "precision_at_100": 0.01089, + "precision_at_1000": 0.00111, + "mrr_at_1": 0.629553, + "mrr_at_3": 0.699272, + "mrr_at_5": 0.711446, + "mrr_at_10": 0.718882, + "mrr_at_20": 0.72114, + "mrr_at_100": 0.722878, + "mrr_at_1000": 0.722952, + "nauc_ndcg_at_1_max": 0.468909, + "nauc_ndcg_at_1_std": -0.120512, + "nauc_ndcg_at_1_diff1": 0.773278, + "nauc_ndcg_at_3_max": 0.466081, + "nauc_ndcg_at_3_std": -0.147874, + "nauc_ndcg_at_3_diff1": 0.725735, + "nauc_ndcg_at_5_max": 0.483513, + "nauc_ndcg_at_5_std": -0.134974, + "nauc_ndcg_at_5_diff1": 0.724338, + "nauc_ndcg_at_10_max": 0.48417, + "nauc_ndcg_at_10_std": -0.138653, + "nauc_ndcg_at_10_diff1": 0.726357, + "nauc_ndcg_at_20_max": 0.485074, + "nauc_ndcg_at_20_std": -0.130313, + "nauc_ndcg_at_20_diff1": 0.728804, + "nauc_ndcg_at_100_max": 0.481135, + "nauc_ndcg_at_100_std": -0.129585, + "nauc_ndcg_at_100_diff1": 0.734781, + "nauc_ndcg_at_1000_max": 0.477898, + "nauc_ndcg_at_1000_std": -0.13356, + "nauc_ndcg_at_1000_diff1": 0.734939, + "nauc_map_at_1_max": 0.415067, + "nauc_map_at_1_std": -0.132336, + "nauc_map_at_1_diff1": 0.754346, + "nauc_map_at_3_max": 0.459804, + "nauc_map_at_3_std": -0.144819, + "nauc_map_at_3_diff1": 0.736585, + "nauc_map_at_5_max": 0.46965, + "nauc_map_at_5_std": -0.139251, + "nauc_map_at_5_diff1": 0.735949, + "nauc_map_at_10_max": 0.469415, + "nauc_map_at_10_std": -0.141926, + "nauc_map_at_10_diff1": 0.736561, + "nauc_map_at_20_max": 0.469638, + "nauc_map_at_20_std": -0.139695, + "nauc_map_at_20_diff1": 0.737503, + "nauc_map_at_100_max": 0.469268, + "nauc_map_at_100_std": -0.139059, + "nauc_map_at_100_diff1": 0.738163, + "nauc_map_at_1000_max": 0.469152, + "nauc_map_at_1000_std": -0.139165, + "nauc_map_at_1000_diff1": 0.738149, + "nauc_recall_at_1_max": 0.415067, + "nauc_recall_at_1_std": -0.132336, + "nauc_recall_at_1_diff1": 0.754346, + "nauc_recall_at_3_max": 0.470715, + "nauc_recall_at_3_std": -0.162622, + "nauc_recall_at_3_diff1": 0.677988, + "nauc_recall_at_5_max": 0.521891, + "nauc_recall_at_5_std": -0.122463, + "nauc_recall_at_5_diff1": 0.659141, + "nauc_recall_at_10_max": 0.546999, + "nauc_recall_at_10_std": -0.130569, + "nauc_recall_at_10_diff1": 0.652552, + "nauc_recall_at_20_max": 0.575029, + "nauc_recall_at_20_std": -0.049824, + "nauc_recall_at_20_diff1": 0.63652, + "nauc_recall_at_100_max": 0.774195, + "nauc_recall_at_100_std": 0.265169, + "nauc_recall_at_100_diff1": 0.690402, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.468909, + "nauc_precision_at_1_std": -0.120512, + "nauc_precision_at_1_diff1": 0.773278, + "nauc_precision_at_3_max": 0.452923, + "nauc_precision_at_3_std": -0.088739, + "nauc_precision_at_3_diff1": 0.499602, + "nauc_precision_at_5_max": 0.442589, + "nauc_precision_at_5_std": -0.032303, + "nauc_precision_at_5_diff1": 0.385843, + "nauc_precision_at_10_max": 0.377099, + "nauc_precision_at_10_std": -0.012707, + "nauc_precision_at_10_diff1": 0.252631, + "nauc_precision_at_20_max": 0.315497, + "nauc_precision_at_20_std": 0.052551, + "nauc_precision_at_20_diff1": 0.14153, + "nauc_precision_at_100_max": 0.122927, + "nauc_precision_at_100_std": 0.125596, + "nauc_precision_at_100_diff1": -0.131289, + "nauc_precision_at_1000_max": 0.047469, + "nauc_precision_at_1000_std": 0.10024, + "nauc_precision_at_1000_diff1": -0.19954, + "nauc_mrr_at_1_max": 0.468909, + "nauc_mrr_at_1_std": -0.120512, + "nauc_mrr_at_1_diff1": 0.773278, + "nauc_mrr_at_3_max": 0.472163, + "nauc_mrr_at_3_std": -0.138745, + "nauc_mrr_at_3_diff1": 0.740181, + "nauc_mrr_at_5_max": 0.479513, + "nauc_mrr_at_5_std": -0.130543, + "nauc_mrr_at_5_diff1": 0.740057, + "nauc_mrr_at_10_max": 0.478652, + "nauc_mrr_at_10_std": -0.131853, + "nauc_mrr_at_10_diff1": 0.741763, + "nauc_mrr_at_20_max": 0.478303, + "nauc_mrr_at_20_std": -0.130968, + "nauc_mrr_at_20_diff1": 0.742115, + "nauc_mrr_at_100_max": 0.477824, + "nauc_mrr_at_100_std": -0.130743, + "nauc_mrr_at_100_diff1": 0.742818, + "nauc_mrr_at_1000_max": 0.477736, + "nauc_mrr_at_1000_std": -0.130855, + "nauc_mrr_at_1000_diff1": 0.742831, + "hit_rate_at_1": 0.62955, + "hit_rate_at_3": 0.78356, + "hit_rate_at_5": 0.83559, + "hit_rate_at_10": 0.88866, + "hit_rate_at_20": 0.92196, + "hit_rate_at_100": 0.98855, + "hit_rate_at_1000": 1.0, + "main_score": 0.75065, + "hf_subset": "sw", + "languages": [ + "swa-Latn" + ] + }, + { + "ndcg_at_1": 0.70222, + "ndcg_at_3": 0.7717, + "ndcg_at_5": 0.79497, + "ndcg_at_10": 0.8132, + "ndcg_at_20": 0.82111, + "ndcg_at_100": 0.82735, + "ndcg_at_1000": 0.82815, + "map_at_1": 0.68241, + "map_at_3": 0.75185, + "map_at_5": 0.76569, + "map_at_10": 0.77337, + "map_at_20": 0.7757, + "map_at_100": 0.77668, + "map_at_1000": 0.77672, + "recall_at_1": 0.68241, + "recall_at_3": 0.82111, + "recall_at_5": 0.87611, + "recall_at_10": 0.93167, + "recall_at_20": 0.96222, + "recall_at_100": 0.99407, + "recall_at_1000": 1.0, + "accuracy": 0.68241, + "precision_at_1": 0.70222, + "precision_at_3": 0.28704, + "precision_at_5": 0.18422, + "precision_at_10": 0.09778, + "precision_at_20": 0.05056, + "precision_at_100": 0.01047, + "precision_at_1000": 0.00105, + "mrr_at_1": 0.702222, + "mrr_at_3": 0.76, + "mrr_at_5": 0.771667, + "mrr_at_10": 0.779224, + "mrr_at_20": 0.781167, + "mrr_at_100": 0.782006, + "mrr_at_1000": 0.782039, + "nauc_ndcg_at_1_max": 0.424328, + "nauc_ndcg_at_1_std": -0.27536, + "nauc_ndcg_at_1_diff1": 0.784137, + "nauc_ndcg_at_3_max": 0.436971, + "nauc_ndcg_at_3_std": -0.359136, + "nauc_ndcg_at_3_diff1": 0.752864, + "nauc_ndcg_at_5_max": 0.441259, + "nauc_ndcg_at_5_std": -0.374583, + "nauc_ndcg_at_5_diff1": 0.752009, + "nauc_ndcg_at_10_max": 0.448161, + "nauc_ndcg_at_10_std": -0.356397, + "nauc_ndcg_at_10_diff1": 0.759724, + "nauc_ndcg_at_20_max": 0.443974, + "nauc_ndcg_at_20_std": -0.347647, + "nauc_ndcg_at_20_diff1": 0.761119, + "nauc_ndcg_at_100_max": 0.440153, + "nauc_ndcg_at_100_std": -0.340073, + "nauc_ndcg_at_100_diff1": 0.762836, + "nauc_ndcg_at_1000_max": 0.440011, + "nauc_ndcg_at_1000_std": -0.339914, + "nauc_ndcg_at_1000_diff1": 0.763622, + "nauc_map_at_1_max": 0.387836, + "nauc_map_at_1_std": -0.274532, + "nauc_map_at_1_diff1": 0.777998, + "nauc_map_at_3_max": 0.423554, + "nauc_map_at_3_std": -0.343072, + "nauc_map_at_3_diff1": 0.76239, + "nauc_map_at_5_max": 0.42898, + "nauc_map_at_5_std": -0.349687, + "nauc_map_at_5_diff1": 0.762105, + "nauc_map_at_10_max": 0.431664, + "nauc_map_at_10_std": -0.341587, + "nauc_map_at_10_diff1": 0.76596, + "nauc_map_at_20_max": 0.430364, + "nauc_map_at_20_std": -0.339293, + "nauc_map_at_20_diff1": 0.765928, + "nauc_map_at_100_max": 0.43019, + "nauc_map_at_100_std": -0.338008, + "nauc_map_at_100_diff1": 0.766416, + "nauc_map_at_1000_max": 0.430165, + "nauc_map_at_1000_std": -0.337992, + "nauc_map_at_1000_diff1": 0.76643, + "nauc_recall_at_1_max": 0.387836, + "nauc_recall_at_1_std": -0.274532, + "nauc_recall_at_1_diff1": 0.777998, + "nauc_recall_at_3_max": 0.447954, + "nauc_recall_at_3_std": -0.433698, + "nauc_recall_at_3_diff1": 0.708488, + "nauc_recall_at_5_max": 0.463877, + "nauc_recall_at_5_std": -0.519874, + "nauc_recall_at_5_diff1": 0.694839, + "nauc_recall_at_10_max": 0.525597, + "nauc_recall_at_10_std": -0.500197, + "nauc_recall_at_10_diff1": 0.698352, + "nauc_recall_at_20_max": 0.533325, + "nauc_recall_at_20_std": -0.4809, + "nauc_recall_at_20_diff1": 0.698632, + "nauc_recall_at_100_max": 0.427754, + "nauc_recall_at_100_std": -0.363387, + "nauc_recall_at_100_diff1": 0.570874, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.424328, + "nauc_precision_at_1_std": -0.27536, + "nauc_precision_at_1_diff1": 0.784137, + "nauc_precision_at_3_max": 0.404052, + "nauc_precision_at_3_std": -0.346129, + "nauc_precision_at_3_diff1": 0.49204, + "nauc_precision_at_5_max": 0.377752, + "nauc_precision_at_5_std": -0.325775, + "nauc_precision_at_5_diff1": 0.355184, + "nauc_precision_at_10_max": 0.378618, + "nauc_precision_at_10_std": -0.227764, + "nauc_precision_at_10_diff1": 0.222168, + "nauc_precision_at_20_max": 0.292419, + "nauc_precision_at_20_std": -0.116457, + "nauc_precision_at_20_diff1": 0.050165, + "nauc_precision_at_100_max": 0.178019, + "nauc_precision_at_100_std": 0.066258, + "nauc_precision_at_100_diff1": -0.200464, + "nauc_precision_at_1000_max": 0.154794, + "nauc_precision_at_1000_std": 0.085111, + "nauc_precision_at_1000_diff1": -0.247602, + "nauc_mrr_at_1_max": 0.424328, + "nauc_mrr_at_1_std": -0.27536, + "nauc_mrr_at_1_diff1": 0.784137, + "nauc_mrr_at_3_max": 0.451725, + "nauc_mrr_at_3_std": -0.324179, + "nauc_mrr_at_3_diff1": 0.762493, + "nauc_mrr_at_5_max": 0.447154, + "nauc_mrr_at_5_std": -0.332614, + "nauc_mrr_at_5_diff1": 0.762968, + "nauc_mrr_at_10_max": 0.448774, + "nauc_mrr_at_10_std": -0.325138, + "nauc_mrr_at_10_diff1": 0.765378, + "nauc_mrr_at_20_max": 0.447798, + "nauc_mrr_at_20_std": -0.323079, + "nauc_mrr_at_20_diff1": 0.766126, + "nauc_mrr_at_100_max": 0.446954, + "nauc_mrr_at_100_std": -0.322266, + "nauc_mrr_at_100_diff1": 0.766339, + "nauc_mrr_at_1000_max": 0.446933, + "nauc_mrr_at_1000_std": -0.32223, + "nauc_mrr_at_1000_diff1": 0.766351, + "hit_rate_at_1": 0.70222, + "hit_rate_at_3": 0.83111, + "hit_rate_at_5": 0.88111, + "hit_rate_at_10": 0.93667, + "hit_rate_at_20": 0.96444, + "hit_rate_at_100": 0.99444, + "hit_rate_at_1000": 1.0, + "main_score": 0.8132, + "hf_subset": "hi", + "languages": [ + "hin-Deva" + ] + }, + { + "ndcg_at_1": 0.67314, + "ndcg_at_3": 0.76571, + "ndcg_at_5": 0.78955, + "ndcg_at_10": 0.80634, + "ndcg_at_20": 0.81359, + "ndcg_at_100": 0.8201, + "ndcg_at_1000": 0.82086, + "map_at_1": 0.64709, + "map_at_3": 0.73763, + "map_at_5": 0.75209, + "map_at_10": 0.76022, + "map_at_20": 0.76232, + "map_at_100": 0.76341, + "map_at_1000": 0.76345, + "recall_at_1": 0.64709, + "recall_at_3": 0.83061, + "recall_at_5": 0.88611, + "recall_at_10": 0.93463, + "recall_at_20": 0.96221, + "recall_at_100": 0.99455, + "recall_at_1000": 1.0, + "accuracy": 0.64709, + "precision_at_1": 0.67314, + "precision_at_3": 0.29554, + "precision_at_5": 0.19019, + "precision_at_10": 0.10143, + "precision_at_20": 0.0523, + "precision_at_100": 0.01086, + "precision_at_1000": 0.00109, + "mrr_at_1": 0.673136, + "mrr_at_3": 0.75383, + "mrr_at_5": 0.765526, + "mrr_at_10": 0.77073, + "mrr_at_20": 0.772706, + "mrr_at_100": 0.773523, + "mrr_at_1000": 0.773556, + "nauc_ndcg_at_1_max": 0.402538, + "nauc_ndcg_at_1_std": -0.2531, + "nauc_ndcg_at_1_diff1": 0.733607, + "nauc_ndcg_at_3_max": 0.425991, + "nauc_ndcg_at_3_std": -0.319442, + "nauc_ndcg_at_3_diff1": 0.699502, + "nauc_ndcg_at_5_max": 0.428806, + "nauc_ndcg_at_5_std": -0.315261, + "nauc_ndcg_at_5_diff1": 0.695596, + "nauc_ndcg_at_10_max": 0.44145, + "nauc_ndcg_at_10_std": -0.306315, + "nauc_ndcg_at_10_diff1": 0.698981, + "nauc_ndcg_at_20_max": 0.436794, + "nauc_ndcg_at_20_std": -0.308243, + "nauc_ndcg_at_20_diff1": 0.704491, + "nauc_ndcg_at_100_max": 0.432777, + "nauc_ndcg_at_100_std": -0.295624, + "nauc_ndcg_at_100_diff1": 0.707118, + "nauc_ndcg_at_1000_max": 0.430855, + "nauc_ndcg_at_1000_std": -0.29801, + "nauc_ndcg_at_1000_diff1": 0.706899, + "nauc_map_at_1_max": 0.36971, + "nauc_map_at_1_std": -0.261775, + "nauc_map_at_1_diff1": 0.740785, + "nauc_map_at_3_max": 0.416408, + "nauc_map_at_3_std": -0.311391, + "nauc_map_at_3_diff1": 0.714277, + "nauc_map_at_5_max": 0.418525, + "nauc_map_at_5_std": -0.308725, + "nauc_map_at_5_diff1": 0.712174, + "nauc_map_at_10_max": 0.424951, + "nauc_map_at_10_std": -0.30421, + "nauc_map_at_10_diff1": 0.712956, + "nauc_map_at_20_max": 0.423627, + "nauc_map_at_20_std": -0.304603, + "nauc_map_at_20_diff1": 0.714208, + "nauc_map_at_100_max": 0.423198, + "nauc_map_at_100_std": -0.302618, + "nauc_map_at_100_diff1": 0.714504, + "nauc_map_at_1000_max": 0.423129, + "nauc_map_at_1000_std": -0.302685, + "nauc_map_at_1000_diff1": 0.714504, + "nauc_recall_at_1_max": 0.36971, + "nauc_recall_at_1_std": -0.261775, + "nauc_recall_at_1_diff1": 0.740785, + "nauc_recall_at_3_max": 0.437143, + "nauc_recall_at_3_std": -0.377541, + "nauc_recall_at_3_diff1": 0.645686, + "nauc_recall_at_5_max": 0.453864, + "nauc_recall_at_5_std": -0.389838, + "nauc_recall_at_5_diff1": 0.618192, + "nauc_recall_at_10_max": 0.55997, + "nauc_recall_at_10_std": -0.374258, + "nauc_recall_at_10_diff1": 0.606129, + "nauc_recall_at_20_max": 0.576621, + "nauc_recall_at_20_std": -0.458812, + "nauc_recall_at_20_diff1": 0.63967, + "nauc_recall_at_100_max": 0.836414, + "nauc_recall_at_100_std": 0.230209, + "nauc_recall_at_100_diff1": 0.765916, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.402538, + "nauc_precision_at_1_std": -0.2531, + "nauc_precision_at_1_diff1": 0.733607, + "nauc_precision_at_3_max": 0.394085, + "nauc_precision_at_3_std": -0.254972, + "nauc_precision_at_3_diff1": 0.36921, + "nauc_precision_at_5_max": 0.331005, + "nauc_precision_at_5_std": -0.168423, + "nauc_precision_at_5_diff1": 0.189889, + "nauc_precision_at_10_max": 0.250924, + "nauc_precision_at_10_std": -0.025071, + "nauc_precision_at_10_diff1": -0.02116, + "nauc_precision_at_20_max": 0.184488, + "nauc_precision_at_20_std": 0.035822, + "nauc_precision_at_20_diff1": -0.128557, + "nauc_precision_at_100_max": 0.072708, + "nauc_precision_at_100_std": 0.184369, + "nauc_precision_at_100_diff1": -0.290752, + "nauc_precision_at_1000_max": 0.043354, + "nauc_precision_at_1000_std": 0.181096, + "nauc_precision_at_1000_diff1": -0.324444, + "nauc_mrr_at_1_max": 0.402538, + "nauc_mrr_at_1_std": -0.2531, + "nauc_mrr_at_1_diff1": 0.733607, + "nauc_mrr_at_3_max": 0.427172, + "nauc_mrr_at_3_std": -0.2822, + "nauc_mrr_at_3_diff1": 0.698536, + "nauc_mrr_at_5_max": 0.425547, + "nauc_mrr_at_5_std": -0.279671, + "nauc_mrr_at_5_diff1": 0.697414, + "nauc_mrr_at_10_max": 0.427212, + "nauc_mrr_at_10_std": -0.280185, + "nauc_mrr_at_10_diff1": 0.700984, + "nauc_mrr_at_20_max": 0.425957, + "nauc_mrr_at_20_std": -0.281322, + "nauc_mrr_at_20_diff1": 0.702394, + "nauc_mrr_at_100_max": 0.425393, + "nauc_mrr_at_100_std": -0.279792, + "nauc_mrr_at_100_diff1": 0.702828, + "nauc_mrr_at_1000_max": 0.425324, + "nauc_mrr_at_1000_std": -0.279856, + "nauc_mrr_at_1000_diff1": 0.70282, + "hit_rate_at_1": 0.67314, + "hit_rate_at_3": 0.85087, + "hit_rate_at_5": 0.90092, + "hit_rate_at_10": 0.93973, + "hit_rate_at_20": 0.96731, + "hit_rate_at_100": 0.99489, + "hit_rate_at_1000": 1.0, + "main_score": 0.80634, + "hf_subset": "ur", + "languages": [ + "urd-Arab" + ] + } + ] + }, + "evaluation_time": 333.26152658462524, + "kg_co2_emissions": null, + "date": 1777885691.978796 +} \ No newline at end of file diff --git a/results/microsoft__harrier-oss-v1-0.6b/f9b9dc8d367d443f2479d27aa5d8d2850c0774ee/LexRetrieval.v1.json b/results/microsoft__harrier-oss-v1-0.6b/f9b9dc8d367d443f2479d27aa5d8d2850c0774ee/LexRetrieval.v1.json new file mode 100644 index 0000000000..3a843eed81 --- /dev/null +++ b/results/microsoft__harrier-oss-v1-0.6b/f9b9dc8d367d443f2479d27aa5d8d2850c0774ee/LexRetrieval.v1.json @@ -0,0 +1,167 @@ +{ + "dataset_revision": "b5b22dee3a30bb32ac579c14afe9eb152e8c70c7", + "task_name": "LexRetrieval.v1", + "mteb_version": "2.12.38", + "scores": { + "test": [ + { + "ndcg_at_1": 0.92175, + "ndcg_at_3": 0.94605, + "ndcg_at_5": 0.94941, + "ndcg_at_10": 0.95181, + "ndcg_at_20": 0.95324, + "ndcg_at_100": 0.95503, + "ndcg_at_1000": 0.9558, + "map_at_1": 0.92175, + "map_at_3": 0.94048, + "map_at_5": 0.94236, + "map_at_10": 0.94336, + "map_at_20": 0.94375, + "map_at_100": 0.94401, + "map_at_1000": 0.94404, + "recall_at_1": 0.92175, + "recall_at_3": 0.96199, + "recall_at_5": 0.97011, + "recall_at_10": 0.97747, + "recall_at_20": 0.98313, + "recall_at_100": 0.99263, + "recall_at_1000": 0.99861, + "accuracy": 0.92175, + "precision_at_1": 0.92175, + "precision_at_3": 0.32066, + "precision_at_5": 0.19402, + "precision_at_10": 0.09775, + "precision_at_20": 0.04916, + "precision_at_100": 0.00993, + "precision_at_1000": 0.001, + "mrr_at_1": 0.921747, + "mrr_at_3": 0.940483, + "mrr_at_5": 0.942356, + "mrr_at_10": 0.943362, + "mrr_at_20": 0.94375, + "mrr_at_100": 0.944009, + "mrr_at_1000": 0.94404, + "nauc_ndcg_at_1_max": 0.832591, + "nauc_ndcg_at_1_std": 0.064713, + "nauc_ndcg_at_1_diff1": 0.937585, + "nauc_ndcg_at_3_max": 0.882851, + "nauc_ndcg_at_3_std": 0.143317, + "nauc_ndcg_at_3_diff1": 0.941614, + "nauc_ndcg_at_5_max": 0.880459, + "nauc_ndcg_at_5_std": 0.136531, + "nauc_ndcg_at_5_diff1": 0.942242, + "nauc_ndcg_at_10_max": 0.877992, + "nauc_ndcg_at_10_std": 0.134872, + "nauc_ndcg_at_10_diff1": 0.941582, + "nauc_ndcg_at_20_max": 0.876277, + "nauc_ndcg_at_20_std": 0.134453, + "nauc_ndcg_at_20_diff1": 0.942042, + "nauc_ndcg_at_100_max": 0.873226, + "nauc_ndcg_at_100_std": 0.132455, + "nauc_ndcg_at_100_diff1": 0.941704, + "nauc_ndcg_at_1000_max": 0.871437, + "nauc_ndcg_at_1000_std": 0.129025, + "nauc_ndcg_at_1000_diff1": 0.941481, + "nauc_map_at_1_max": 0.832591, + "nauc_map_at_1_std": 0.064713, + "nauc_map_at_1_diff1": 0.937585, + "nauc_map_at_3_max": 0.869218, + "nauc_map_at_3_std": 0.120253, + "nauc_map_at_3_diff1": 0.940717, + "nauc_map_at_5_max": 0.86761, + "nauc_map_at_5_std": 0.116462, + "nauc_map_at_5_diff1": 0.940973, + "nauc_map_at_10_max": 0.866526, + "nauc_map_at_10_std": 0.11547, + "nauc_map_at_10_diff1": 0.940716, + "nauc_map_at_20_max": 0.866036, + "nauc_map_at_20_std": 0.115499, + "nauc_map_at_20_diff1": 0.940775, + "nauc_map_at_100_max": 0.865656, + "nauc_map_at_100_std": 0.115208, + "nauc_map_at_100_diff1": 0.940742, + "nauc_map_at_1000_max": 0.865596, + "nauc_map_at_1000_std": 0.115123, + "nauc_map_at_1000_diff1": 0.940736, + "nauc_recall_at_1_max": 0.832591, + "nauc_recall_at_1_std": 0.064713, + "nauc_recall_at_1_diff1": 0.937585, + "nauc_recall_at_3_max": 0.942971, + "nauc_recall_at_3_std": 0.246343, + "nauc_recall_at_3_diff1": 0.945423, + "nauc_recall_at_5_max": 0.949458, + "nauc_recall_at_5_std": 0.244322, + "nauc_recall_at_5_diff1": 0.949198, + "nauc_recall_at_10_max": 0.955551, + "nauc_recall_at_10_std": 0.269242, + "nauc_recall_at_10_diff1": 0.947142, + "nauc_recall_at_20_max": 0.963257, + "nauc_recall_at_20_std": 0.303607, + "nauc_recall_at_20_diff1": 0.955057, + "nauc_recall_at_100_max": 0.974356, + "nauc_recall_at_100_std": 0.449145, + "nauc_recall_at_100_diff1": 0.959674, + "nauc_recall_at_1000_max": 0.965705, + "nauc_recall_at_1000_std": 0.891041, + "nauc_recall_at_1000_diff1": 0.978633, + "nauc_precision_at_1_max": 0.832591, + "nauc_precision_at_1_std": 0.064713, + "nauc_precision_at_1_diff1": 0.937585, + "nauc_precision_at_3_max": 0.942971, + "nauc_precision_at_3_std": 0.246343, + "nauc_precision_at_3_diff1": 0.945423, + "nauc_precision_at_5_max": 0.949458, + "nauc_precision_at_5_std": 0.244322, + "nauc_precision_at_5_diff1": 0.949198, + "nauc_precision_at_10_max": 0.955551, + "nauc_precision_at_10_std": 0.269242, + "nauc_precision_at_10_diff1": 0.947142, + "nauc_precision_at_20_max": 0.963257, + "nauc_precision_at_20_std": 0.303607, + "nauc_precision_at_20_diff1": 0.955057, + "nauc_precision_at_100_max": 0.974356, + "nauc_precision_at_100_std": 0.449145, + "nauc_precision_at_100_diff1": 0.959674, + "nauc_precision_at_1000_max": 0.965705, + "nauc_precision_at_1000_std": 0.891041, + "nauc_precision_at_1000_diff1": 0.978633, + "nauc_mrr_at_1_max": 0.832591, + "nauc_mrr_at_1_std": 0.064713, + "nauc_mrr_at_1_diff1": 0.937585, + "nauc_mrr_at_3_max": 0.869218, + "nauc_mrr_at_3_std": 0.120253, + "nauc_mrr_at_3_diff1": 0.940717, + "nauc_mrr_at_5_max": 0.86761, + "nauc_mrr_at_5_std": 0.116462, + "nauc_mrr_at_5_diff1": 0.940973, + "nauc_mrr_at_10_max": 0.866526, + "nauc_mrr_at_10_std": 0.11547, + "nauc_mrr_at_10_diff1": 0.940716, + "nauc_mrr_at_20_max": 0.866036, + "nauc_mrr_at_20_std": 0.115499, + "nauc_mrr_at_20_diff1": 0.940775, + "nauc_mrr_at_100_max": 0.865656, + "nauc_mrr_at_100_std": 0.115208, + "nauc_mrr_at_100_diff1": 0.940742, + "nauc_mrr_at_1000_max": 0.865596, + "nauc_mrr_at_1000_std": 0.115123, + "nauc_mrr_at_1000_diff1": 0.940736, + "hit_rate_at_1": 0.92175, + "hit_rate_at_3": 0.96199, + "hit_rate_at_5": 0.97011, + "hit_rate_at_10": 0.97747, + "hit_rate_at_20": 0.98313, + "hit_rate_at_100": 0.99263, + "hit_rate_at_1000": 0.99861, + "main_score": 0.95181, + "hf_subset": "default", + "languages": [ + "dan-Latn" + ] + } + ] + }, + "evaluation_time": 136.2393193244934, + "kg_co2_emissions": null, + "date": 1777885343.693943 +} \ No newline at end of file diff --git a/results/microsoft__harrier-oss-v1-0.6b/f9b9dc8d367d443f2479d27aa5d8d2850c0774ee/PublicNewsRetrieval.json b/results/microsoft__harrier-oss-v1-0.6b/f9b9dc8d367d443f2479d27aa5d8d2850c0774ee/PublicNewsRetrieval.json new file mode 100644 index 0000000000..3187ce1fa6 --- /dev/null +++ b/results/microsoft__harrier-oss-v1-0.6b/f9b9dc8d367d443f2479d27aa5d8d2850c0774ee/PublicNewsRetrieval.json @@ -0,0 +1,322 @@ +{ + "dataset_revision": "82dad8c469646feae9631d69ba92bebfc0a586e9", + "task_name": "PublicNewsRetrieval", + "mteb_version": "2.12.37", + "scores": { + "test": [ + { + "ndcg_at_1": 0.79692, + "ndcg_at_3": 0.84892, + "ndcg_at_5": 0.85836, + "ndcg_at_10": 0.86762, + "ndcg_at_20": 0.87299, + "ndcg_at_100": 0.87863, + "ndcg_at_1000": 0.88088, + "map_at_1": 0.78558, + "map_at_3": 0.83417, + "map_at_5": 0.83994, + "map_at_10": 0.84404, + "map_at_20": 0.84569, + "map_at_100": 0.84662, + "map_at_1000": 0.84673, + "recall_at_1": 0.78558, + "recall_at_3": 0.885, + "recall_at_5": 0.90706, + "recall_at_10": 0.93465, + "recall_at_20": 0.95454, + "recall_at_100": 0.9828, + "recall_at_1000": 0.99933, + "accuracy": 0.78558, + "precision_at_1": 0.79692, + "precision_at_3": 0.3034, + "precision_at_5": 0.18727, + "precision_at_10": 0.09672, + "precision_at_20": 0.0496, + "precision_at_100": 0.01027, + "precision_at_1000": 0.00105, + "mrr_at_1": 0.796917, + "mrr_at_3": 0.840147, + "mrr_at_5": 0.84504, + "mrr_at_10": 0.848613, + "mrr_at_20": 0.850033, + "mrr_at_100": 0.850754, + "mrr_at_1000": 0.850842, + "nauc_ndcg_at_1_max": 0.513952, + "nauc_ndcg_at_1_std": -0.347863, + "nauc_ndcg_at_1_diff1": 0.868078, + "nauc_ndcg_at_3_max": 0.573838, + "nauc_ndcg_at_3_std": -0.328864, + "nauc_ndcg_at_3_diff1": 0.842991, + "nauc_ndcg_at_5_max": 0.580925, + "nauc_ndcg_at_5_std": -0.304244, + "nauc_ndcg_at_5_diff1": 0.842056, + "nauc_ndcg_at_10_max": 0.575118, + "nauc_ndcg_at_10_std": -0.315637, + "nauc_ndcg_at_10_diff1": 0.845642, + "nauc_ndcg_at_20_max": 0.579502, + "nauc_ndcg_at_20_std": -0.309066, + "nauc_ndcg_at_20_diff1": 0.847809, + "nauc_ndcg_at_100_max": 0.575014, + "nauc_ndcg_at_100_std": -0.312395, + "nauc_ndcg_at_100_diff1": 0.851826, + "nauc_ndcg_at_1000_max": 0.570165, + "nauc_ndcg_at_1000_std": -0.318844, + "nauc_ndcg_at_1000_diff1": 0.851636, + "nauc_map_at_1_max": 0.49426, + "nauc_map_at_1_std": -0.353792, + "nauc_map_at_1_diff1": 0.874964, + "nauc_map_at_3_max": 0.555566, + "nauc_map_at_3_std": -0.337961, + "nauc_map_at_3_diff1": 0.852468, + "nauc_map_at_5_max": 0.558789, + "nauc_map_at_5_std": -0.326622, + "nauc_map_at_5_diff1": 0.852233, + "nauc_map_at_10_max": 0.557247, + "nauc_map_at_10_std": -0.330781, + "nauc_map_at_10_diff1": 0.853788, + "nauc_map_at_20_max": 0.558723, + "nauc_map_at_20_std": -0.329187, + "nauc_map_at_20_diff1": 0.854278, + "nauc_map_at_100_max": 0.558318, + "nauc_map_at_100_std": -0.329923, + "nauc_map_at_100_diff1": 0.854955, + "nauc_map_at_1000_max": 0.558174, + "nauc_map_at_1000_std": -0.330032, + "nauc_map_at_1000_diff1": 0.854951, + "nauc_recall_at_1_max": 0.49426, + "nauc_recall_at_1_std": -0.353792, + "nauc_recall_at_1_diff1": 0.874964, + "nauc_recall_at_3_max": 0.637801, + "nauc_recall_at_3_std": -0.295034, + "nauc_recall_at_3_diff1": 0.808412, + "nauc_recall_at_5_max": 0.669439, + "nauc_recall_at_5_std": -0.202646, + "nauc_recall_at_5_diff1": 0.793795, + "nauc_recall_at_10_max": 0.666262, + "nauc_recall_at_10_std": -0.232882, + "nauc_recall_at_10_diff1": 0.796852, + "nauc_recall_at_20_max": 0.736529, + "nauc_recall_at_20_std": -0.140327, + "nauc_recall_at_20_diff1": 0.796248, + "nauc_recall_at_100_max": 0.822056, + "nauc_recall_at_100_std": 0.067503, + "nauc_recall_at_100_diff1": 0.841406, + "nauc_recall_at_1000_max": 0.869301, + "nauc_recall_at_1000_std": 0.358037, + "nauc_recall_at_1000_diff1": 0.722291, + "nauc_precision_at_1_max": 0.513952, + "nauc_precision_at_1_std": -0.347863, + "nauc_precision_at_1_diff1": 0.868078, + "nauc_precision_at_3_max": 0.529197, + "nauc_precision_at_3_std": -0.180566, + "nauc_precision_at_3_diff1": 0.527095, + "nauc_precision_at_5_max": 0.488566, + "nauc_precision_at_5_std": -0.06222, + "nauc_precision_at_5_diff1": 0.408518, + "nauc_precision_at_10_max": 0.404101, + "nauc_precision_at_10_std": -0.018306, + "nauc_precision_at_10_diff1": 0.255233, + "nauc_precision_at_20_max": 0.34513, + "nauc_precision_at_20_std": 0.120703, + "nauc_precision_at_20_diff1": 0.088064, + "nauc_precision_at_100_max": 0.134455, + "nauc_precision_at_100_std": 0.259305, + "nauc_precision_at_100_diff1": -0.196003, + "nauc_precision_at_1000_max": -0.04749, + "nauc_precision_at_1000_std": 0.261034, + "nauc_precision_at_1000_diff1": -0.411568, + "nauc_mrr_at_1_max": 0.513952, + "nauc_mrr_at_1_std": -0.347863, + "nauc_mrr_at_1_diff1": 0.868078, + "nauc_mrr_at_3_max": 0.560154, + "nauc_mrr_at_3_std": -0.333288, + "nauc_mrr_at_3_diff1": 0.848145, + "nauc_mrr_at_5_max": 0.562844, + "nauc_mrr_at_5_std": -0.320802, + "nauc_mrr_at_5_diff1": 0.848043, + "nauc_mrr_at_10_max": 0.557897, + "nauc_mrr_at_10_std": -0.326209, + "nauc_mrr_at_10_diff1": 0.849139, + "nauc_mrr_at_20_max": 0.558598, + "nauc_mrr_at_20_std": -0.32534, + "nauc_mrr_at_20_diff1": 0.850093, + "nauc_mrr_at_100_max": 0.55779, + "nauc_mrr_at_100_std": -0.32684, + "nauc_mrr_at_100_diff1": 0.850575, + "nauc_mrr_at_1000_max": 0.557665, + "nauc_mrr_at_1000_std": -0.326969, + "nauc_mrr_at_1000_diff1": 0.850593, + "hit_rate_at_1": 0.79692, + "hit_rate_at_3": 0.89276, + "hit_rate_at_5": 0.91354, + "hit_rate_at_10": 0.93968, + "hit_rate_at_20": 0.95912, + "hit_rate_at_100": 0.98458, + "hit_rate_at_1000": 0.99933, + "main_score": 0.86762, + "hf_subset": "pl", + "languages": [ + "pol-Latn" + ] + }, + { + "ndcg_at_1": 0.71111, + "ndcg_at_3": 0.78676, + "ndcg_at_5": 0.806, + "ndcg_at_10": 0.8215, + "ndcg_at_20": 0.8289, + "ndcg_at_100": 0.83585, + "ndcg_at_1000": 0.83688, + "map_at_1": 0.69188, + "map_at_3": 0.76552, + "map_at_5": 0.77714, + "map_at_10": 0.78403, + "map_at_20": 0.78613, + "map_at_100": 0.7873, + "map_at_1000": 0.78735, + "recall_at_1": 0.69188, + "recall_at_3": 0.83869, + "recall_at_5": 0.88408, + "recall_at_10": 0.93002, + "recall_at_20": 0.95855, + "recall_at_100": 0.99283, + "recall_at_1000": 1.0, + "accuracy": 0.69188, + "precision_at_1": 0.71111, + "precision_at_3": 0.29377, + "precision_at_5": 0.18667, + "precision_at_10": 0.09863, + "precision_at_20": 0.05092, + "precision_at_100": 0.01061, + "precision_at_1000": 0.00107, + "mrr_at_1": 0.711111, + "mrr_at_3": 0.774389, + "mrr_at_5": 0.783775, + "mrr_at_10": 0.789826, + "mrr_at_20": 0.791788, + "mrr_at_100": 0.792707, + "mrr_at_1000": 0.792748, + "nauc_ndcg_at_1_max": 0.31449, + "nauc_ndcg_at_1_std": -0.492099, + "nauc_ndcg_at_1_diff1": 0.830803, + "nauc_ndcg_at_3_max": 0.335681, + "nauc_ndcg_at_3_std": -0.578422, + "nauc_ndcg_at_3_diff1": 0.807321, + "nauc_ndcg_at_5_max": 0.336225, + "nauc_ndcg_at_5_std": -0.583893, + "nauc_ndcg_at_5_diff1": 0.804385, + "nauc_ndcg_at_10_max": 0.341701, + "nauc_ndcg_at_10_std": -0.567381, + "nauc_ndcg_at_10_diff1": 0.808378, + "nauc_ndcg_at_20_max": 0.332472, + "nauc_ndcg_at_20_std": -0.567234, + "nauc_ndcg_at_20_diff1": 0.809181, + "nauc_ndcg_at_100_max": 0.333248, + "nauc_ndcg_at_100_std": -0.554076, + "nauc_ndcg_at_100_diff1": 0.81175, + "nauc_ndcg_at_1000_max": 0.332844, + "nauc_ndcg_at_1000_std": -0.552193, + "nauc_ndcg_at_1000_diff1": 0.811995, + "nauc_map_at_1_max": 0.291915, + "nauc_map_at_1_std": -0.477362, + "nauc_map_at_1_diff1": 0.829621, + "nauc_map_at_3_max": 0.326042, + "nauc_map_at_3_std": -0.554535, + "nauc_map_at_3_diff1": 0.813532, + "nauc_map_at_5_max": 0.327346, + "nauc_map_at_5_std": -0.557533, + "nauc_map_at_5_diff1": 0.812329, + "nauc_map_at_10_max": 0.329784, + "nauc_map_at_10_std": -0.55077, + "nauc_map_at_10_diff1": 0.813916, + "nauc_map_at_20_max": 0.327665, + "nauc_map_at_20_std": -0.550695, + "nauc_map_at_20_diff1": 0.814187, + "nauc_map_at_100_max": 0.327689, + "nauc_map_at_100_std": -0.548358, + "nauc_map_at_100_diff1": 0.814488, + "nauc_map_at_1000_max": 0.327675, + "nauc_map_at_1000_std": -0.54826, + "nauc_map_at_1000_diff1": 0.8145, + "nauc_recall_at_1_max": 0.291915, + "nauc_recall_at_1_std": -0.477362, + "nauc_recall_at_1_diff1": 0.829621, + "nauc_recall_at_3_max": 0.356, + "nauc_recall_at_3_std": -0.670882, + "nauc_recall_at_3_diff1": 0.778525, + "nauc_recall_at_5_max": 0.360755, + "nauc_recall_at_5_std": -0.725181, + "nauc_recall_at_5_diff1": 0.761733, + "nauc_recall_at_10_max": 0.408175, + "nauc_recall_at_10_std": -0.702427, + "nauc_recall_at_10_diff1": 0.76184, + "nauc_recall_at_20_max": 0.323315, + "nauc_recall_at_20_std": -0.79929, + "nauc_recall_at_20_diff1": 0.745988, + "nauc_recall_at_100_max": 0.396514, + "nauc_recall_at_100_std": -0.788282, + "nauc_recall_at_100_diff1": 0.763599, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.31449, + "nauc_precision_at_1_std": -0.492099, + "nauc_precision_at_1_diff1": 0.830803, + "nauc_precision_at_3_max": 0.321261, + "nauc_precision_at_3_std": -0.492027, + "nauc_precision_at_3_diff1": 0.481948, + "nauc_precision_at_5_max": 0.286161, + "nauc_precision_at_5_std": -0.41672, + "nauc_precision_at_5_diff1": 0.327397, + "nauc_precision_at_10_max": 0.254837, + "nauc_precision_at_10_std": -0.232936, + "nauc_precision_at_10_diff1": 0.137384, + "nauc_precision_at_20_max": 0.173352, + "nauc_precision_at_20_std": -0.126188, + "nauc_precision_at_20_diff1": -0.027592, + "nauc_precision_at_100_max": 0.088207, + "nauc_precision_at_100_std": 0.175994, + "nauc_precision_at_100_diff1": -0.287656, + "nauc_precision_at_1000_max": 0.059539, + "nauc_precision_at_1000_std": 0.240918, + "nauc_precision_at_1000_diff1": -0.351598, + "nauc_mrr_at_1_max": 0.31449, + "nauc_mrr_at_1_std": -0.492099, + "nauc_mrr_at_1_diff1": 0.830803, + "nauc_mrr_at_3_max": 0.334488, + "nauc_mrr_at_3_std": -0.550255, + "nauc_mrr_at_3_diff1": 0.813669, + "nauc_mrr_at_5_max": 0.333859, + "nauc_mrr_at_5_std": -0.549951, + "nauc_mrr_at_5_diff1": 0.812711, + "nauc_mrr_at_10_max": 0.33446, + "nauc_mrr_at_10_std": -0.543311, + "nauc_mrr_at_10_diff1": 0.814546, + "nauc_mrr_at_20_max": 0.33193, + "nauc_mrr_at_20_std": -0.542915, + "nauc_mrr_at_20_diff1": 0.814664, + "nauc_mrr_at_100_max": 0.331666, + "nauc_mrr_at_100_std": -0.541769, + "nauc_mrr_at_100_diff1": 0.814945, + "nauc_mrr_at_1000_max": 0.331649, + "nauc_mrr_at_1000_std": -0.541694, + "nauc_mrr_at_1000_diff1": 0.814964, + "hit_rate_at_1": 0.71111, + "hit_rate_at_3": 0.84917, + "hit_rate_at_5": 0.89078, + "hit_rate_at_10": 0.93475, + "hit_rate_at_20": 0.96312, + "hit_rate_at_100": 0.99385, + "hit_rate_at_1000": 1.0, + "main_score": 0.8215, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ] + } + ] + }, + "evaluation_time": 376.67814779281616, + "kg_co2_emissions": null, + "date": 1777965240.64847 +} \ No newline at end of file diff --git a/results/microsoft__harrier-oss-v1-0.6b/f9b9dc8d367d443f2479d27aa5d8d2850c0774ee/model_meta.json b/results/microsoft__harrier-oss-v1-0.6b/f9b9dc8d367d443f2479d27aa5d8d2850c0774ee/model_meta.json index 963fbc1c8b..bc2c74ed3f 100644 --- a/results/microsoft__harrier-oss-v1-0.6b/f9b9dc8d367d443f2479d27aa5d8d2850c0774ee/model_meta.json +++ b/results/microsoft__harrier-oss-v1-0.6b/f9b9dc8d367d443f2479d27aa5d8d2850c0774ee/model_meta.json @@ -1,19 +1,19 @@ { "loader_kwargs": { - "instruction_template": "", + "instruction_template": "", "apply_instruction_to_passages": false, "prompts_dict": { - "AILAStatutes-query": "Identifying the most relevant statutes for a given situation", + "AILAStatutes": "Identifying the most relevant statutes for a given situation", "AfriSentiClassification": "Given a text, categorized by sentiment into positive, negative, or neutral", "AlloProfClusteringS2S.v2": "Identify the topic of document titles from Allo Prof dataset", - "AlloprofReranking-query": "Given a question, retrieve passages that answer the question", + "AlloprofReranking": "Given a question, retrieve passages that answer the question", "AmazonCounterfactualClassification": "Given an Amazon review, judge whether it is counterfactual.", "ArXivHierarchicalClusteringP2P": "Identify the main and secondary category of Arxiv papers based on the titles and abstracts", "ArXivHierarchicalClusteringS2S": "Identify the main and secondary category of Arxiv papers based on the titles", - "ArguAna-query": "Given a claim, find documents that refute the claim", + "ArguAna": "Given a claim, find documents that refute the claim", "ArmenianParaphrasePC": "Retrieve semantically similar text", "BUCC.v2": "Retrieve parallel sentences", - "BelebeleRetrieval-query": "Retrieval the relevant passage for the given query", + "BelebeleRetrieval": "Retrieval the relevant passage for the given query", "BibleNLPBitextMining": "Retrieve parallel sentences", "BigPatentClustering.v2": "Identify the category of documents from the Big Patent dataset", "BiorxivClusteringP2P.v2": "Identify the main category of Biorxiv papers based on the titles and abstracts", @@ -25,8 +25,8 @@ "CSFDSKMovieReviewSentimentClassification": "Given a movie review, classify its rating on a scale from 0 to 5", "CTKFactsNLI": "Retrieve semantically similar text", "CataloniaTweetClassification": "Given a tweet, classify its sentiment into AGAINST, FAVOR or NEUTRAL towards Catalonia's independence.", - "Core17InstructionRetrieval-query": "Retrieve relevant passages for the given query with conditions", - "CovidRetrieval-query": "Given a question on COVID-19, retrieve news articles that answer the question", + "Core17InstructionRetrieval": "Retrieve relevant passages for the given query with conditions", + "CovidRetrieval": "Given a question on COVID-19, retrieve news articles that answer the question", "CyrillicTurkicLangClassification": "Given a text, classify its language", "CzechProductReviewSentimentClassification": "Classify product reviews into positive, neutral, or negative sentiment", "DBpediaClassification": "Given the following text, retrieve the appropriate DBpedia category including Company, EducationalInstitution, Artist, Athlete, OfficeHolder, MeanOfTransportation, Building, NaturalPlace, Village, Animal, Plant, Album, Film, WrittenWork.", @@ -42,7 +42,7 @@ "GreekLegalCodeClassification": "Given a greek legal text, classify its topic", "GujaratiNewsClassification": "Given a Gujarati news articles, classify ist topic", "HALClusteringS2S.v2": "Identify the topic of titles from HAL", - "HagridRetrieval-query": "Given a question, retrieve relevant responses", + "HagridRetrieval": "Given a question, retrieve relevant responses", "IN22GenBitextMining": "Retrieve parallel sentences", "IndicCrosslingualSTS": "Retrieve semantically similar text", "IndicGenBenchFloresBitextMining": "Retrieve parallel sentences", @@ -54,10 +54,10 @@ "KorHateSpeechMLClassification": "Given a Korean online news comments, classify its fine-grained hate speech classes", "KorSarcasmClassification": "Given a twitter, categorized it into sarcasm or not_sarcasm", "KurdishSentimentClassification": "Given a text, categorized by sentiment into positive or negative", - "LEMBPasskeyRetrieval-query": "Retrieval the relevant passage for the given query", - "LegalBenchCorporateLobbying-query": "Given a query, retrieve relevant legal bill summaries", - "MIRACLRetrievalHardNegatives-query": "Retrieve Wikipedia passages that answer the question", - "MLQARetrieval-query": "Retrieval the relevant passage for the given query", + "LEMBPasskeyRetrieval": "Retrieval the relevant passage for the given query", + "LegalBenchCorporateLobbying": "Given a query, retrieve relevant legal bill summaries", + "MIRACLRetrievalHardNegatives": "Retrieve Wikipedia passages that answer the question", + "MLQARetrieval": "Retrieval the relevant passage for the given query", "MacedonianTweetSentimentClassification": "Given a Macedonian tweet, categorized by sentiment into positive, negative, or neutral", "MalteseNewsClassification": "Given a maltese new, classify its topic", "MasakhaNEWSClassification": "Classify the News in the given texts into one of the seven category: politics,sports,health,business,entertainment,technology,religion ", @@ -68,7 +68,7 @@ "MultiHateClassification": "Given a text, categorized by sentiment into hate or non-hate", "NTREXBitextMining": "Retrieve parallel sentences", "NepaliNewsClassification": "Given a news article, categorized it into business, entertainment or sports", - "News21InstructionRetrieval-query": "Retrieve relevant passages for the given query with conditions", + "News21InstructionRetrieval": "Retrieve relevant passages for the given query with conditions", "NollySentiBitextMining": "Retrieve parallel sentences", "NordicLangClassification": "Given a text in a Nordic language, classify the language into one of the following categories: Danish, Swedish, Norwegian (Nynorsk), Norwegian (Bokm\u00e5l), Faroese, Icelandic.", "NorwegianCourtsBitextMining": "Retrieve parallel sentences", @@ -86,10 +86,10 @@ "PpcPC": "Retrieve semantically similar text", "PunjabiNewsClassification": "Given a news article, categorized it into two-classes", "RTE3": "Retrieve semantically similar text", - "Robust04InstructionRetrieval-query": "Retrieve relevant passages for the given query with conditions", + "Robust04InstructionRetrieval": "Retrieve relevant passages for the given query with conditions", "RomaniBibleClustering": "Identify verses from the Bible in Kalderash Romani by book.", - "RuBQReranking-query": "Given a question, retrieve Wikipedia passages that answer the question", - "SCIDOCS-query": "Given a scientific paper title, retrieve paper abstracts that are cited by the given paper", + "RuBQReranking": "Given a question, retrieve Wikipedia passages that answer the question", + "SCIDOCS": "Given a scientific paper title, retrieve paper abstracts that are cited by the given paper", "SIB200ClusteringS2S": "Identify the category of documents", "SICK-R": "Retrieve semantically similar text", "STS12": "Retrieve semantically related sentences", @@ -107,31 +107,31 @@ "SinhalaNewsClassification": "Given a news article, categorized it into political, business, technology, sports and Entertainment", "SiswatiNewsClassification": "Identify fine-grained news categories in Siswati language.", "SlovakMovieReviewSentimentClassification": "Given a movie review, categorized it into positive or negative", - "SpartQA-query": "Given the following spatial reasoning question, retrieve the right answer.", + "SpartQA": "Given the following spatial reasoning question, retrieve the right answer.", "SprintDuplicateQuestions": "Find questions that have the same meaning as the input question", "StackExchangeClustering.v2": "Identify the topic or theme of StackExchange posts based on the titles", - "StackOverflowQA-query": "Given a question about coding, retrieval code or passage that can solve user's question", - "StatcanDialogueDatasetRetrieval-query": "Retrieval the relevant passage for the given query", + "StackOverflowQA": "Given a question about coding, retrieval code or passage that can solve user's question", + "StatcanDialogueDatasetRetrieval": "Retrieval the relevant passage for the given query", "SwahiliNewsClassification": "Given a news article, classify its domain", "SwednClusteringP2P": "Identify news categories in Swedish passages", "SwissJudgementClassification": "Given a news article, categorized it into approval or dismissal", - "T2Reranking-query": "Given a Chinese search query, retrieve web passages that answer the question", + "T2Reranking": "Given a Chinese search query, retrieve web passages that answer the question", "TERRa": "Given a premise, retrieve a hypothesis that is entailed by the premise", - "TRECCOVID-query": "Given a medical query, retrieve documents that answer the query", + "TRECCOVID": "Given a medical query, retrieve documents that answer the query", "Tatoeba": "Retrieve parallel sentences", - "TempReasonL1-query": "Given the following question about time, retrieve the correct answer.", + "TempReasonL1": "Given the following question about time, retrieve the correct answer.", "ToxicConversationsClassification": "Classify the given comments as either toxic or not toxic", "TswanaNewsClassification": "Given a news article, classify its topic", "TweetTopicSingleClassification": "Gvien a twitter, classify its topic", - "TwitterHjerneRetrieval-query": "Retrieve answers to questions asked in Danish tweets", + "TwitterHjerneRetrieval": "Retrieve answers to questions asked in Danish tweets", "TwitterURLCorpus": "Find tweets that have the same meaning as the input tweet", - "VoyageMMarcoReranking-query": "Given a Japanese search query, retrieve web passages that answer the question", - "WebLINXCandidatesReranking-query": "Retrieval the relevant passage for the given query", + "VoyageMMarcoReranking": "Given a Japanese search query, retrieve web passages that answer the question", + "WebLINXCandidatesReranking": "Retrieval the relevant passage for the given query", "WikiCitiesClustering": "Identify of Wikipedia articles of cities by country", "WikiClusteringP2P.v2": "Identify the category of wiki passages", - "WikipediaRerankingMultilingual-query": "Retrieval the relevant passage for the given query", - "WikipediaRetrievalMultilingual-query": "Retrieval the relevant passage for the given query", - "WinoGrande-query": "Given the following sentence, retrieve an appropriate answer to fill in the missing underscored part.", + "WikipediaRerankingMultilingual": "Retrieval the relevant passage for the given query", + "WikipediaRetrievalMultilingual": "Retrieval the relevant passage for the given query", + "WinoGrande": "Given the following sentence, retrieve an appropriate answer to fill in the missing underscored part.", "XNLI": "Retrieve semantically similar text", "indonli": "Retrieve semantically similar text" } @@ -258,58 +258,58 @@ "similarity_fn_name": "cosine", "use_instructions": true, "training_datasets": [ + "NordicLangClassification", "TempReasonL1", - "CodeSearchNetCCRetrieval", - "FiQA2018", - "CSFDSKMovieReviewSentimentClassification", "ScalaClassification", - "TERRa", - "AppsRetrieval", - "CodeFeedbackST", - "PAC", - "CataloniaTweetClassification", - "HellaSwag", - "PawsXPairClassification", - "Quail", - "AlphaNLI", - "CodeSearchNetRetrieval", - "CyrillicTurkicLangClassification", - "DBpediaClassification", + "HotpotQAHardNegatives", + "SciFact", + "NFCorpus", + "GermanSTSBenchmark", "SemRel24STS", - "SwissJudgementClassification", + "SpartQA", + "COIRCodeSearchNetRetrieval", + "OdiaNewsClassification", + "AmazonCounterfactualClassification", + "CodeTransOceanDLRetrieval", "MrTidyRetrieval", - "StatcanDialogueDatasetRetrieval", - "PoemSentimentClassification", + "WinoGrande", + "ToxicConversationsClassification", + "NQ", "SyntheticText2SQLRetrieval", - "MSMARCO", - "QuoraRetrieval", - "GreekLegalCodeClassification", - "NFCorpus", - "MIRACLRetrieval", + "STSB", + "PawsXPairClassification", + "CyrillicTurkicLangClassification", "BornholmBitextMining", - "CodeTransOceanContestRetrieval", - "CosQARetrieval", - "CodeFeedbackMT", - "NQ", + "AlphaNLI", + "IndicLangClassification", + "CodeSearchNetCCRetrieval", + "TERRa", + "MIRACLRetrieval", + "StackOverflowQARetrieval", + "GreekLegalCodeClassification", + "HellaSwag", "FEVER", - "ToxicConversationsClassification", + "CosQARetrieval", + "PoemSentimentClassification", + "FiQA2018", + "EmotionClassification", "NusaParagraphEmotionClassification", - "CodeTransOceanDLRetrieval", + "CataloniaTweetClassification", + "MSMARCO", "CzechProductReviewSentimentClassification", - "AmazonCounterfactualClassification", - "NordicLangClassification", - "STSB", - "SciFact", - "IndicLangClassification", + "SwissJudgementClassification", + "CodeTransOceanContestRetrieval", + "DBpediaClassification", + "PAC", + "Quail", + "CodeFeedbackST", + "CodeSearchNetRetrieval", + "CodeFeedbackMT", + "QuoraRetrieval", "FinParaSTS", - "StackOverflowQARetrieval", - "COIRCodeSearchNetRetrieval", - "SpartQA", - "WinoGrande", - "HotpotQAHardNegatives", - "OdiaNewsClassification", - "EmotionClassification", - "GermanSTSBenchmark" + "StatcanDialogueDatasetRetrieval", + "AppsRetrieval", + "CSFDSKMovieReviewSentimentClassification" ], "adapted_from": "Qwen/Qwen3-0.6B", "superseded_by": null, @@ -323,6 +323,7 @@ "contacts": null, "experiment_kwargs": null, "output_dtypes": null, + "extra_requirements_groups": null, "loader": "InstructSentenceTransformerModel", "is_cross_encoder": false } \ No newline at end of file diff --git a/results/nvidia__llama-nemotron-embed-1b-v2/b4caa8456edd360b3b4e938d94ed4398dd437fad/experiments/trust_remote_code_True/ElasticKBRetrieval.json b/results/nvidia__llama-nemotron-embed-1b-v2/b4caa8456edd360b3b4e938d94ed4398dd437fad/experiments/trust_remote_code_True/ElasticKBRetrieval.json new file mode 100644 index 0000000000..14f0f30dbb --- /dev/null +++ b/results/nvidia__llama-nemotron-embed-1b-v2/b4caa8456edd360b3b4e938d94ed4398dd437fad/experiments/trust_remote_code_True/ElasticKBRetrieval.json @@ -0,0 +1,324 @@ +{ + "dataset_revision": "21bdf1e024bf7c9f46720017559ce2f8c6116507", + "task_name": "ElasticKBRetrieval", + "mteb_version": "2.12.38", + "scores": { + "synthetic_test": [ + { + "ndcg_at_1": 0.81383, + "ndcg_at_3": 0.66086, + "ndcg_at_5": 0.61717, + "ndcg_at_10": 0.58396, + "ndcg_at_20": 0.58167, + "ndcg_at_100": 0.62705, + "ndcg_at_1000": 0.68439, + "map_at_1": 0.26823, + "map_at_3": 0.33277, + "map_at_5": 0.35444, + "map_at_10": 0.37923, + "map_at_20": 0.40429, + "map_at_100": 0.43715, + "map_at_1000": 0.44875, + "recall_at_1": 0.26823, + "recall_at_3": 0.35394, + "recall_at_5": 0.38812, + "recall_at_10": 0.45021, + "recall_at_20": 0.52263, + "recall_at_100": 0.69937, + "recall_at_1000": 0.87633, + "accuracy": 0.26823, + "precision_at_1": 0.81383, + "precision_at_3": 0.50355, + "precision_at_5": 0.3883, + "precision_at_10": 0.26755, + "precision_at_20": 0.19096, + "precision_at_100": 0.0734, + "precision_at_1000": 0.01309, + "mrr_at_1": 0.81383, + "mrr_at_3": 0.882979, + "mrr_at_5": 0.885638, + "mrr_at_10": 0.888002, + "mrr_at_20": 0.888486, + "mrr_at_100": 0.888647, + "mrr_at_1000": 0.888647, + "nauc_ndcg_at_1_max": 0.363299, + "nauc_ndcg_at_1_std": -0.121585, + "nauc_ndcg_at_1_diff1": 0.672869, + "nauc_ndcg_at_3_max": 0.156246, + "nauc_ndcg_at_3_std": -0.014678, + "nauc_ndcg_at_3_diff1": 0.200232, + "nauc_ndcg_at_5_max": 0.18912, + "nauc_ndcg_at_5_std": -0.006627, + "nauc_ndcg_at_5_diff1": 0.257664, + "nauc_ndcg_at_10_max": 0.187569, + "nauc_ndcg_at_10_std": -0.041069, + "nauc_ndcg_at_10_diff1": 0.366718, + "nauc_ndcg_at_20_max": 0.215397, + "nauc_ndcg_at_20_std": -0.00908, + "nauc_ndcg_at_20_diff1": 0.398781, + "nauc_ndcg_at_100_max": 0.289426, + "nauc_ndcg_at_100_std": 0.104107, + "nauc_ndcg_at_100_diff1": 0.364457, + "nauc_ndcg_at_1000_max": 0.248936, + "nauc_ndcg_at_1000_std": 0.126512, + "nauc_ndcg_at_1000_diff1": 0.322564, + "nauc_map_at_1_max": 0.168887, + "nauc_map_at_1_std": -0.137863, + "nauc_map_at_1_diff1": 0.499945, + "nauc_map_at_3_max": 0.163496, + "nauc_map_at_3_std": -0.145952, + "nauc_map_at_3_diff1": 0.457335, + "nauc_map_at_5_max": 0.187886, + "nauc_map_at_5_std": -0.128173, + "nauc_map_at_5_diff1": 0.449166, + "nauc_map_at_10_max": 0.197586, + "nauc_map_at_10_std": -0.113408, + "nauc_map_at_10_diff1": 0.447516, + "nauc_map_at_20_max": 0.215686, + "nauc_map_at_20_std": -0.065349, + "nauc_map_at_20_diff1": 0.430703, + "nauc_map_at_100_max": 0.229064, + "nauc_map_at_100_std": 0.001863, + "nauc_map_at_100_diff1": 0.404105, + "nauc_map_at_1000_max": 0.221086, + "nauc_map_at_1000_std": 0.00847, + "nauc_map_at_1000_diff1": 0.397797, + "nauc_recall_at_1_max": 0.168887, + "nauc_recall_at_1_std": -0.137863, + "nauc_recall_at_1_diff1": 0.499945, + "nauc_recall_at_3_max": 0.153208, + "nauc_recall_at_3_std": -0.154402, + "nauc_recall_at_3_diff1": 0.435434, + "nauc_recall_at_5_max": 0.182496, + "nauc_recall_at_5_std": -0.131652, + "nauc_recall_at_5_diff1": 0.422477, + "nauc_recall_at_10_max": 0.180243, + "nauc_recall_at_10_std": -0.124504, + "nauc_recall_at_10_diff1": 0.41827, + "nauc_recall_at_20_max": 0.21705, + "nauc_recall_at_20_std": -0.048591, + "nauc_recall_at_20_diff1": 0.367208, + "nauc_recall_at_100_max": 0.303978, + "nauc_recall_at_100_std": 0.195326, + "nauc_recall_at_100_diff1": 0.263344, + "nauc_recall_at_1000_max": 0.140624, + "nauc_recall_at_1000_std": 0.295759, + "nauc_recall_at_1000_diff1": 0.070542, + "nauc_precision_at_1_max": 0.363299, + "nauc_precision_at_1_std": -0.121585, + "nauc_precision_at_1_diff1": 0.672869, + "nauc_precision_at_3_max": 0.030229, + "nauc_precision_at_3_std": 0.133531, + "nauc_precision_at_3_diff1": -0.261001, + "nauc_precision_at_5_max": 0.05217, + "nauc_precision_at_5_std": 0.210879, + "nauc_precision_at_5_diff1": -0.287166, + "nauc_precision_at_10_max": -0.026858, + "nauc_precision_at_10_std": 0.246244, + "nauc_precision_at_10_diff1": -0.321373, + "nauc_precision_at_20_max": -0.045295, + "nauc_precision_at_20_std": 0.319143, + "nauc_precision_at_20_diff1": -0.341202, + "nauc_precision_at_100_max": -0.146893, + "nauc_precision_at_100_std": 0.276587, + "nauc_precision_at_100_diff1": -0.365133, + "nauc_precision_at_1000_max": -0.269033, + "nauc_precision_at_1000_std": 0.07898, + "nauc_precision_at_1000_diff1": -0.336617, + "nauc_mrr_at_1_max": 0.363299, + "nauc_mrr_at_1_std": -0.121585, + "nauc_mrr_at_1_diff1": 0.672869, + "nauc_mrr_at_3_max": 0.404675, + "nauc_mrr_at_3_std": -0.124034, + "nauc_mrr_at_3_diff1": 0.672923, + "nauc_mrr_at_5_max": 0.40084, + "nauc_mrr_at_5_std": -0.107415, + "nauc_mrr_at_5_diff1": 0.668329, + "nauc_mrr_at_10_max": 0.395898, + "nauc_mrr_at_10_std": -0.111685, + "nauc_mrr_at_10_diff1": 0.671566, + "nauc_mrr_at_20_max": 0.394411, + "nauc_mrr_at_20_std": -0.116162, + "nauc_mrr_at_20_diff1": 0.670664, + "nauc_mrr_at_100_max": 0.393688, + "nauc_mrr_at_100_std": -0.116077, + "nauc_mrr_at_100_diff1": 0.670842, + "nauc_mrr_at_1000_max": 0.393688, + "nauc_mrr_at_1000_std": -0.116077, + "nauc_mrr_at_1000_diff1": 0.670842, + "hit_rate_at_1": 0.81383, + "hit_rate_at_3": 0.96277, + "hit_rate_at_5": 0.9734, + "hit_rate_at_10": 0.98936, + "hit_rate_at_20": 0.99468, + "hit_rate_at_100": 1.0, + "hit_rate_at_1000": 1.0, + "main_score": 0.58396, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ], + "real_chat_test": [ + { + "ndcg_at_1": 0.86207, + "ndcg_at_3": 0.77372, + "ndcg_at_5": 0.72257, + "ndcg_at_10": 0.66686, + "ndcg_at_20": 0.62301, + "ndcg_at_100": 0.56131, + "ndcg_at_1000": 0.63234, + "map_at_1": 0.0704, + "map_at_3": 0.12361, + "map_at_5": 0.14987, + "map_at_10": 0.18175, + "map_at_20": 0.21048, + "map_at_100": 0.27359, + "map_at_1000": 0.33679, + "recall_at_1": 0.0704, + "recall_at_3": 0.13045, + "recall_at_5": 0.16499, + "recall_at_10": 0.21735, + "recall_at_20": 0.27423, + "recall_at_100": 0.44803, + "recall_at_1000": 0.75931, + "accuracy": 0.0704, + "precision_at_1": 0.86207, + "precision_at_3": 0.73132, + "precision_at_5": 0.65086, + "precision_at_10": 0.55, + "precision_at_20": 0.45345, + "precision_at_100": 0.26207, + "precision_at_1000": 0.08771, + "mrr_at_1": 0.862069, + "mrr_at_3": 0.905891, + "mrr_at_5": 0.90977, + "mrr_at_10": 0.910386, + "mrr_at_20": 0.910386, + "mrr_at_100": 0.910735, + "mrr_at_1000": 0.910735, + "nauc_ndcg_at_1_max": 0.564464, + "nauc_ndcg_at_1_std": 0.277272, + "nauc_ndcg_at_1_diff1": 0.401162, + "nauc_ndcg_at_3_max": 0.316651, + "nauc_ndcg_at_3_std": 0.171123, + "nauc_ndcg_at_3_diff1": 0.107428, + "nauc_ndcg_at_5_max": 0.283112, + "nauc_ndcg_at_5_std": 0.178061, + "nauc_ndcg_at_5_diff1": 0.033808, + "nauc_ndcg_at_10_max": 0.248533, + "nauc_ndcg_at_10_std": 0.197385, + "nauc_ndcg_at_10_diff1": -0.019863, + "nauc_ndcg_at_20_max": 0.242055, + "nauc_ndcg_at_20_std": 0.215817, + "nauc_ndcg_at_20_diff1": -0.046249, + "nauc_ndcg_at_100_max": 0.304336, + "nauc_ndcg_at_100_std": 0.261595, + "nauc_ndcg_at_100_diff1": 0.016958, + "nauc_ndcg_at_1000_max": 0.350186, + "nauc_ndcg_at_1000_std": 0.356215, + "nauc_ndcg_at_1000_diff1": 0.064669, + "nauc_map_at_1_max": 0.264333, + "nauc_map_at_1_std": -0.075808, + "nauc_map_at_1_diff1": 0.268427, + "nauc_map_at_3_max": 0.22279, + "nauc_map_at_3_std": -0.076977, + "nauc_map_at_3_diff1": 0.190494, + "nauc_map_at_5_max": 0.229857, + "nauc_map_at_5_std": -0.060983, + "nauc_map_at_5_diff1": 0.157238, + "nauc_map_at_10_max": 0.237259, + "nauc_map_at_10_std": -0.03423, + "nauc_map_at_10_diff1": 0.147229, + "nauc_map_at_20_max": 0.262474, + "nauc_map_at_20_std": 0.002125, + "nauc_map_at_20_diff1": 0.131283, + "nauc_map_at_100_max": 0.30564, + "nauc_map_at_100_std": 0.128848, + "nauc_map_at_100_diff1": 0.104901, + "nauc_map_at_1000_max": 0.320065, + "nauc_map_at_1000_std": 0.218303, + "nauc_map_at_1000_diff1": 0.054348, + "nauc_recall_at_1_max": 0.264333, + "nauc_recall_at_1_std": -0.075808, + "nauc_recall_at_1_diff1": 0.268427, + "nauc_recall_at_3_max": 0.201923, + "nauc_recall_at_3_std": -0.085576, + "nauc_recall_at_3_diff1": 0.178306, + "nauc_recall_at_5_max": 0.19932, + "nauc_recall_at_5_std": -0.078759, + "nauc_recall_at_5_diff1": 0.140225, + "nauc_recall_at_10_max": 0.193039, + "nauc_recall_at_10_std": -0.048732, + "nauc_recall_at_10_diff1": 0.129294, + "nauc_recall_at_20_max": 0.219641, + "nauc_recall_at_20_std": -0.008336, + "nauc_recall_at_20_diff1": 0.107202, + "nauc_recall_at_100_max": 0.244679, + "nauc_recall_at_100_std": 0.16875, + "nauc_recall_at_100_diff1": 0.093282, + "nauc_recall_at_1000_max": 0.240979, + "nauc_recall_at_1000_std": 0.338454, + "nauc_recall_at_1000_diff1": 0.067323, + "nauc_precision_at_1_max": 0.564464, + "nauc_precision_at_1_std": 0.277272, + "nauc_precision_at_1_diff1": 0.401162, + "nauc_precision_at_3_max": 0.193258, + "nauc_precision_at_3_std": 0.172758, + "nauc_precision_at_3_diff1": -0.001389, + "nauc_precision_at_5_max": 0.168204, + "nauc_precision_at_5_std": 0.207653, + "nauc_precision_at_5_diff1": -0.073379, + "nauc_precision_at_10_max": 0.097052, + "nauc_precision_at_10_std": 0.22928, + "nauc_precision_at_10_diff1": -0.143411, + "nauc_precision_at_20_max": 0.043404, + "nauc_precision_at_20_std": 0.24306, + "nauc_precision_at_20_diff1": -0.182715, + "nauc_precision_at_100_max": -0.034132, + "nauc_precision_at_100_std": 0.228644, + "nauc_precision_at_100_diff1": -0.193781, + "nauc_precision_at_1000_max": -0.096857, + "nauc_precision_at_1000_std": 0.068177, + "nauc_precision_at_1000_diff1": -0.211707, + "nauc_mrr_at_1_max": 0.564464, + "nauc_mrr_at_1_std": 0.277272, + "nauc_mrr_at_1_diff1": 0.401162, + "nauc_mrr_at_3_max": 0.531526, + "nauc_mrr_at_3_std": 0.229374, + "nauc_mrr_at_3_diff1": 0.437428, + "nauc_mrr_at_5_max": 0.52979, + "nauc_mrr_at_5_std": 0.22737, + "nauc_mrr_at_5_diff1": 0.438324, + "nauc_mrr_at_10_max": 0.541697, + "nauc_mrr_at_10_std": 0.241335, + "nauc_mrr_at_10_diff1": 0.440557, + "nauc_mrr_at_20_max": 0.541697, + "nauc_mrr_at_20_std": 0.241335, + "nauc_mrr_at_20_diff1": 0.440557, + "nauc_mrr_at_100_max": 0.541072, + "nauc_mrr_at_100_std": 0.242638, + "nauc_mrr_at_100_diff1": 0.439133, + "nauc_mrr_at_1000_max": 0.541072, + "nauc_mrr_at_1000_std": 0.242638, + "nauc_mrr_at_1000_diff1": 0.439133, + "hit_rate_at_1": 0.86207, + "hit_rate_at_3": 0.96121, + "hit_rate_at_5": 0.97845, + "hit_rate_at_10": 0.98276, + "hit_rate_at_20": 0.98276, + "hit_rate_at_100": 1.0, + "hit_rate_at_1000": 1.0, + "main_score": 0.66686, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + } + ] + }, + "evaluation_time": 248.4510109424591, + "kg_co2_emissions": null, + "date": 1777842218.834509 +} \ No newline at end of file diff --git a/results/nvidia__llama-nemotron-embed-1b-v2/b4caa8456edd360b3b4e938d94ed4398dd437fad/experiments/trust_remote_code_True/GlobalNewsRetrieval.json b/results/nvidia__llama-nemotron-embed-1b-v2/b4caa8456edd360b3b4e938d94ed4398dd437fad/experiments/trust_remote_code_True/GlobalNewsRetrieval.json new file mode 100644 index 0000000000..72e19aeb52 --- /dev/null +++ b/results/nvidia__llama-nemotron-embed-1b-v2/b4caa8456edd360b3b4e938d94ed4398dd437fad/experiments/trust_remote_code_True/GlobalNewsRetrieval.json @@ -0,0 +1,1562 @@ +{ + "dataset_revision": "234dcd32922d32c78e08bfe506a2b90a5401cf7c", + "task_name": "GlobalNewsRetrieval", + "mteb_version": "2.12.38", + "scores": { + "test": [ + { + "ndcg_at_1": 0.7193, + "ndcg_at_3": 0.79686, + "ndcg_at_5": 0.81923, + "ndcg_at_10": 0.83195, + "ndcg_at_20": 0.83822, + "ndcg_at_100": 0.84465, + "ndcg_at_1000": 0.84505, + "map_at_1": 0.67962, + "map_at_3": 0.76869, + "map_at_5": 0.78284, + "map_at_10": 0.78934, + "map_at_20": 0.79144, + "map_at_100": 0.79251, + "map_at_1000": 0.79253, + "recall_at_1": 0.67962, + "recall_at_3": 0.85589, + "recall_at_5": 0.90664, + "recall_at_10": 0.94215, + "recall_at_20": 0.9647, + "recall_at_100": 0.99708, + "recall_at_1000": 1.0, + "accuracy": 0.67962, + "precision_at_1": 0.7193, + "precision_at_3": 0.31161, + "precision_at_5": 0.2, + "precision_at_10": 0.10526, + "precision_at_20": 0.05426, + "precision_at_100": 0.01128, + "precision_at_1000": 0.00113, + "mrr_at_1": 0.719298, + "mrr_at_3": 0.791562, + "mrr_at_5": 0.802339, + "mrr_at_10": 0.806141, + "mrr_at_20": 0.807422, + "mrr_at_100": 0.80806, + "mrr_at_1000": 0.808072, + "nauc_ndcg_at_1_max": 0.35323, + "nauc_ndcg_at_1_std": -0.240803, + "nauc_ndcg_at_1_diff1": 0.838576, + "nauc_ndcg_at_3_max": 0.360117, + "nauc_ndcg_at_3_std": -0.256486, + "nauc_ndcg_at_3_diff1": 0.793844, + "nauc_ndcg_at_5_max": 0.365368, + "nauc_ndcg_at_5_std": -0.273536, + "nauc_ndcg_at_5_diff1": 0.795418, + "nauc_ndcg_at_10_max": 0.372127, + "nauc_ndcg_at_10_std": -0.270434, + "nauc_ndcg_at_10_diff1": 0.798716, + "nauc_ndcg_at_20_max": 0.373795, + "nauc_ndcg_at_20_std": -0.264819, + "nauc_ndcg_at_20_diff1": 0.804347, + "nauc_ndcg_at_100_max": 0.368612, + "nauc_ndcg_at_100_std": -0.258605, + "nauc_ndcg_at_100_diff1": 0.806057, + "nauc_ndcg_at_1000_max": 0.366915, + "nauc_ndcg_at_1000_std": -0.26178, + "nauc_ndcg_at_1000_diff1": 0.806131, + "nauc_map_at_1_max": 0.278942, + "nauc_map_at_1_std": -0.266438, + "nauc_map_at_1_diff1": 0.821833, + "nauc_map_at_3_max": 0.333446, + "nauc_map_at_3_std": -0.268425, + "nauc_map_at_3_diff1": 0.80097, + "nauc_map_at_5_max": 0.341126, + "nauc_map_at_5_std": -0.27755, + "nauc_map_at_5_diff1": 0.802593, + "nauc_map_at_10_max": 0.34559, + "nauc_map_at_10_std": -0.277322, + "nauc_map_at_10_diff1": 0.804132, + "nauc_map_at_20_max": 0.345938, + "nauc_map_at_20_std": -0.275225, + "nauc_map_at_20_diff1": 0.80551, + "nauc_map_at_100_max": 0.345235, + "nauc_map_at_100_std": -0.274282, + "nauc_map_at_100_diff1": 0.805755, + "nauc_map_at_1000_max": 0.345172, + "nauc_map_at_1000_std": -0.274393, + "nauc_map_at_1000_diff1": 0.80576, + "nauc_recall_at_1_max": 0.278942, + "nauc_recall_at_1_std": -0.266438, + "nauc_recall_at_1_diff1": 0.821833, + "nauc_recall_at_3_max": 0.379265, + "nauc_recall_at_3_std": -0.24284, + "nauc_recall_at_3_diff1": 0.732727, + "nauc_recall_at_5_max": 0.392329, + "nauc_recall_at_5_std": -0.304112, + "nauc_recall_at_5_diff1": 0.707975, + "nauc_recall_at_10_max": 0.450757, + "nauc_recall_at_10_std": -0.305177, + "nauc_recall_at_10_diff1": 0.701006, + "nauc_recall_at_20_max": 0.536201, + "nauc_recall_at_20_std": -0.262375, + "nauc_recall_at_20_diff1": 0.746777, + "nauc_recall_at_100_max": 0.960299, + "nauc_recall_at_100_std": 0.852269, + "nauc_recall_at_100_diff1": 0.785158, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.35323, + "nauc_precision_at_1_std": -0.240803, + "nauc_precision_at_1_diff1": 0.838576, + "nauc_precision_at_3_max": 0.355353, + "nauc_precision_at_3_std": -0.055682, + "nauc_precision_at_3_diff1": 0.312922, + "nauc_precision_at_5_max": 0.310695, + "nauc_precision_at_5_std": -0.033961, + "nauc_precision_at_5_diff1": 0.110882, + "nauc_precision_at_10_max": 0.238862, + "nauc_precision_at_10_std": 0.032446, + "nauc_precision_at_10_diff1": -0.060839, + "nauc_precision_at_20_max": 0.190759, + "nauc_precision_at_20_std": 0.098439, + "nauc_precision_at_20_diff1": -0.152221, + "nauc_precision_at_100_max": 0.115751, + "nauc_precision_at_100_std": 0.179124, + "nauc_precision_at_100_diff1": -0.289532, + "nauc_precision_at_1000_max": 0.100807, + "nauc_precision_at_1000_std": 0.167019, + "nauc_precision_at_1000_diff1": -0.302798, + "nauc_mrr_at_1_max": 0.35323, + "nauc_mrr_at_1_std": -0.240803, + "nauc_mrr_at_1_diff1": 0.838576, + "nauc_mrr_at_3_max": 0.395093, + "nauc_mrr_at_3_std": -0.233187, + "nauc_mrr_at_3_diff1": 0.81254, + "nauc_mrr_at_5_max": 0.390662, + "nauc_mrr_at_5_std": -0.239389, + "nauc_mrr_at_5_diff1": 0.813406, + "nauc_mrr_at_10_max": 0.388931, + "nauc_mrr_at_10_std": -0.237809, + "nauc_mrr_at_10_diff1": 0.813914, + "nauc_mrr_at_20_max": 0.388616, + "nauc_mrr_at_20_std": -0.237786, + "nauc_mrr_at_20_diff1": 0.815304, + "nauc_mrr_at_100_max": 0.388069, + "nauc_mrr_at_100_std": -0.237353, + "nauc_mrr_at_100_diff1": 0.815627, + "nauc_mrr_at_1000_max": 0.388027, + "nauc_mrr_at_1000_std": -0.237433, + "nauc_mrr_at_1000_diff1": 0.815629, + "hit_rate_at_1": 0.7193, + "hit_rate_at_3": 0.88095, + "hit_rate_at_5": 0.92732, + "hit_rate_at_10": 0.95489, + "hit_rate_at_20": 0.97243, + "hit_rate_at_100": 0.99749, + "hit_rate_at_1000": 1.0, + "main_score": 0.83195, + "hf_subset": "ar", + "languages": [ + "ara-Arab" + ] + }, + { + "ndcg_at_1": 0.81613, + "ndcg_at_3": 0.84945, + "ndcg_at_5": 0.86472, + "ndcg_at_10": 0.87346, + "ndcg_at_20": 0.87873, + "ndcg_at_100": 0.88691, + "ndcg_at_1000": 0.88774, + "map_at_1": 0.75278, + "map_at_3": 0.83035, + "map_at_5": 0.84099, + "map_at_10": 0.84531, + "map_at_20": 0.84724, + "map_at_100": 0.84868, + "map_at_1000": 0.84874, + "recall_at_1": 0.75278, + "recall_at_3": 0.87894, + "recall_at_5": 0.91228, + "recall_at_10": 0.93701, + "recall_at_20": 0.9552, + "recall_at_100": 0.99462, + "recall_at_1000": 1.0, + "accuracy": 0.75278, + "precision_at_1": 0.81613, + "precision_at_3": 0.33405, + "precision_at_5": 0.21054, + "precision_at_10": 0.10882, + "precision_at_20": 0.05591, + "precision_at_100": 0.01176, + "precision_at_1000": 0.00118, + "mrr_at_1": 0.816129, + "mrr_at_3": 0.856989, + "mrr_at_5": 0.863172, + "mrr_at_10": 0.865857, + "mrr_at_20": 0.866824, + "mrr_at_100": 0.867622, + "mrr_at_1000": 0.867657, + "nauc_ndcg_at_1_max": 0.43603, + "nauc_ndcg_at_1_std": -0.590004, + "nauc_ndcg_at_1_diff1": 0.82863, + "nauc_ndcg_at_3_max": 0.430842, + "nauc_ndcg_at_3_std": -0.633397, + "nauc_ndcg_at_3_diff1": 0.781458, + "nauc_ndcg_at_5_max": 0.464009, + "nauc_ndcg_at_5_std": -0.638149, + "nauc_ndcg_at_5_diff1": 0.785666, + "nauc_ndcg_at_10_max": 0.456769, + "nauc_ndcg_at_10_std": -0.655232, + "nauc_ndcg_at_10_diff1": 0.791352, + "nauc_ndcg_at_20_max": 0.454233, + "nauc_ndcg_at_20_std": -0.647732, + "nauc_ndcg_at_20_diff1": 0.795686, + "nauc_ndcg_at_100_max": 0.450896, + "nauc_ndcg_at_100_std": -0.628828, + "nauc_ndcg_at_100_diff1": 0.796517, + "nauc_ndcg_at_1000_max": 0.448175, + "nauc_ndcg_at_1000_std": -0.628387, + "nauc_ndcg_at_1000_diff1": 0.796032, + "nauc_map_at_1_max": 0.332411, + "nauc_map_at_1_std": -0.522612, + "nauc_map_at_1_diff1": 0.799732, + "nauc_map_at_3_max": 0.416971, + "nauc_map_at_3_std": -0.617161, + "nauc_map_at_3_diff1": 0.785261, + "nauc_map_at_5_max": 0.442548, + "nauc_map_at_5_std": -0.616145, + "nauc_map_at_5_diff1": 0.787706, + "nauc_map_at_10_max": 0.440139, + "nauc_map_at_10_std": -0.621603, + "nauc_map_at_10_diff1": 0.789632, + "nauc_map_at_20_max": 0.438747, + "nauc_map_at_20_std": -0.619666, + "nauc_map_at_20_diff1": 0.791, + "nauc_map_at_100_max": 0.437973, + "nauc_map_at_100_std": -0.617588, + "nauc_map_at_100_diff1": 0.791145, + "nauc_map_at_1000_max": 0.437831, + "nauc_map_at_1000_std": -0.617537, + "nauc_map_at_1000_diff1": 0.791118, + "nauc_recall_at_1_max": 0.332411, + "nauc_recall_at_1_std": -0.522612, + "nauc_recall_at_1_diff1": 0.799732, + "nauc_recall_at_3_max": 0.432103, + "nauc_recall_at_3_std": -0.676305, + "nauc_recall_at_3_diff1": 0.733135, + "nauc_recall_at_5_max": 0.528344, + "nauc_recall_at_5_std": -0.713051, + "nauc_recall_at_5_diff1": 0.727239, + "nauc_recall_at_10_max": 0.51526, + "nauc_recall_at_10_std": -0.843242, + "nauc_recall_at_10_diff1": 0.746498, + "nauc_recall_at_20_max": 0.530857, + "nauc_recall_at_20_std": -0.862932, + "nauc_recall_at_20_diff1": 0.784004, + "nauc_recall_at_100_max": 0.80112, + "nauc_recall_at_100_std": -0.641923, + "nauc_recall_at_100_diff1": 0.846405, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.43603, + "nauc_precision_at_1_std": -0.590004, + "nauc_precision_at_1_diff1": 0.82863, + "nauc_precision_at_3_max": 0.230987, + "nauc_precision_at_3_std": -0.210679, + "nauc_precision_at_3_diff1": 0.06755, + "nauc_precision_at_5_max": 0.205146, + "nauc_precision_at_5_std": -0.102514, + "nauc_precision_at_5_diff1": -0.056977, + "nauc_precision_at_10_max": 0.138833, + "nauc_precision_at_10_std": -0.049605, + "nauc_precision_at_10_diff1": -0.143995, + "nauc_precision_at_20_max": 0.079022, + "nauc_precision_at_20_std": 0.027675, + "nauc_precision_at_20_diff1": -0.210629, + "nauc_precision_at_100_max": -0.026879, + "nauc_precision_at_100_std": 0.196857, + "nauc_precision_at_100_diff1": -0.36215, + "nauc_precision_at_1000_max": -0.045555, + "nauc_precision_at_1000_std": 0.21631, + "nauc_precision_at_1000_diff1": -0.381823, + "nauc_mrr_at_1_max": 0.43603, + "nauc_mrr_at_1_std": -0.590004, + "nauc_mrr_at_1_diff1": 0.82863, + "nauc_mrr_at_3_max": 0.455475, + "nauc_mrr_at_3_std": -0.626021, + "nauc_mrr_at_3_diff1": 0.809392, + "nauc_mrr_at_5_max": 0.457447, + "nauc_mrr_at_5_std": -0.632699, + "nauc_mrr_at_5_diff1": 0.811834, + "nauc_mrr_at_10_max": 0.453219, + "nauc_mrr_at_10_std": -0.638365, + "nauc_mrr_at_10_diff1": 0.813684, + "nauc_mrr_at_20_max": 0.453225, + "nauc_mrr_at_20_std": -0.635455, + "nauc_mrr_at_20_diff1": 0.813809, + "nauc_mrr_at_100_max": 0.453924, + "nauc_mrr_at_100_std": -0.632044, + "nauc_mrr_at_100_diff1": 0.81414, + "nauc_mrr_at_1000_max": 0.453806, + "nauc_mrr_at_1000_std": -0.632072, + "nauc_mrr_at_1000_diff1": 0.814127, + "hit_rate_at_1": 0.81613, + "hit_rate_at_3": 0.90538, + "hit_rate_at_5": 0.93226, + "hit_rate_at_10": 0.95269, + "hit_rate_at_20": 0.96667, + "hit_rate_at_100": 0.9957, + "hit_rate_at_1000": 1.0, + "main_score": 0.87346, + "hf_subset": "zh", + "languages": [ + "zho-Hans" + ] + }, + { + "ndcg_at_1": 0.75334, + "ndcg_at_3": 0.82043, + "ndcg_at_5": 0.83917, + "ndcg_at_10": 0.85149, + "ndcg_at_20": 0.85912, + "ndcg_at_100": 0.86402, + "ndcg_at_1000": 0.86434, + "map_at_1": 0.72645, + "map_at_3": 0.79892, + "map_at_5": 0.81088, + "map_at_10": 0.81644, + "map_at_20": 0.81898, + "map_at_100": 0.81983, + "map_at_1000": 0.81985, + "recall_at_1": 0.72645, + "recall_at_3": 0.86802, + "recall_at_5": 0.91007, + "recall_at_10": 0.94647, + "recall_at_20": 0.97431, + "recall_at_100": 0.99794, + "recall_at_1000": 1.0, + "accuracy": 0.72645, + "precision_at_1": 0.75334, + "precision_at_3": 0.3073, + "precision_at_5": 0.19568, + "precision_at_10": 0.10226, + "precision_at_20": 0.05298, + "precision_at_100": 0.0109, + "precision_at_1000": 0.00109, + "mrr_at_1": 0.75334, + "mrr_at_3": 0.813806, + "mrr_at_5": 0.822388, + "mrr_at_10": 0.826926, + "mrr_at_20": 0.828482, + "mrr_at_100": 0.829126, + "mrr_at_1000": 0.829132, + "nauc_ndcg_at_1_max": 0.494979, + "nauc_ndcg_at_1_std": -0.388609, + "nauc_ndcg_at_1_diff1": 0.848431, + "nauc_ndcg_at_3_max": 0.524325, + "nauc_ndcg_at_3_std": -0.38746, + "nauc_ndcg_at_3_diff1": 0.823978, + "nauc_ndcg_at_5_max": 0.542951, + "nauc_ndcg_at_5_std": -0.395933, + "nauc_ndcg_at_5_diff1": 0.833259, + "nauc_ndcg_at_10_max": 0.528979, + "nauc_ndcg_at_10_std": -0.413225, + "nauc_ndcg_at_10_diff1": 0.832524, + "nauc_ndcg_at_20_max": 0.522938, + "nauc_ndcg_at_20_std": -0.401194, + "nauc_ndcg_at_20_diff1": 0.832631, + "nauc_ndcg_at_100_max": 0.522143, + "nauc_ndcg_at_100_std": -0.397508, + "nauc_ndcg_at_100_diff1": 0.834618, + "nauc_ndcg_at_1000_max": 0.521153, + "nauc_ndcg_at_1000_std": -0.397582, + "nauc_ndcg_at_1000_diff1": 0.834287, + "nauc_map_at_1_max": 0.462673, + "nauc_map_at_1_std": -0.37691, + "nauc_map_at_1_diff1": 0.839487, + "nauc_map_at_3_max": 0.510598, + "nauc_map_at_3_std": -0.392165, + "nauc_map_at_3_diff1": 0.828793, + "nauc_map_at_5_max": 0.52319, + "nauc_map_at_5_std": -0.39561, + "nauc_map_at_5_diff1": 0.83418, + "nauc_map_at_10_max": 0.517593, + "nauc_map_at_10_std": -0.402286, + "nauc_map_at_10_diff1": 0.833789, + "nauc_map_at_20_max": 0.515561, + "nauc_map_at_20_std": -0.398755, + "nauc_map_at_20_diff1": 0.833576, + "nauc_map_at_100_max": 0.515653, + "nauc_map_at_100_std": -0.398021, + "nauc_map_at_100_diff1": 0.833897, + "nauc_map_at_1000_max": 0.515597, + "nauc_map_at_1000_std": -0.398033, + "nauc_map_at_1000_diff1": 0.83388, + "nauc_recall_at_1_max": 0.462673, + "nauc_recall_at_1_std": -0.37691, + "nauc_recall_at_1_diff1": 0.839487, + "nauc_recall_at_3_max": 0.561789, + "nauc_recall_at_3_std": -0.364691, + "nauc_recall_at_3_diff1": 0.787935, + "nauc_recall_at_5_max": 0.636231, + "nauc_recall_at_5_std": -0.398544, + "nauc_recall_at_5_diff1": 0.809703, + "nauc_recall_at_10_max": 0.596724, + "nauc_recall_at_10_std": -0.532973, + "nauc_recall_at_10_diff1": 0.794958, + "nauc_recall_at_20_max": 0.580455, + "nauc_recall_at_20_std": -0.465337, + "nauc_recall_at_20_diff1": 0.784494, + "nauc_recall_at_100_max": 0.901994, + "nauc_recall_at_100_std": -0.391683, + "nauc_recall_at_100_diff1": 0.967331, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.494979, + "nauc_precision_at_1_std": -0.388609, + "nauc_precision_at_1_diff1": 0.848431, + "nauc_precision_at_3_max": 0.392973, + "nauc_precision_at_3_std": -0.216466, + "nauc_precision_at_3_diff1": 0.395451, + "nauc_precision_at_5_max": 0.304006, + "nauc_precision_at_5_std": -0.111872, + "nauc_precision_at_5_diff1": 0.182769, + "nauc_precision_at_10_max": 0.145552, + "nauc_precision_at_10_std": -0.06517, + "nauc_precision_at_10_diff1": -0.030493, + "nauc_precision_at_20_max": -0.000822, + "nauc_precision_at_20_std": 0.077784, + "nauc_precision_at_20_diff1": -0.211987, + "nauc_precision_at_100_max": -0.099877, + "nauc_precision_at_100_std": 0.173281, + "nauc_precision_at_100_diff1": -0.360103, + "nauc_precision_at_1000_max": -0.115605, + "nauc_precision_at_1000_std": 0.176903, + "nauc_precision_at_1000_diff1": -0.37258, + "nauc_mrr_at_1_max": 0.494979, + "nauc_mrr_at_1_std": -0.388609, + "nauc_mrr_at_1_diff1": 0.848431, + "nauc_mrr_at_3_max": 0.52144, + "nauc_mrr_at_3_std": -0.389306, + "nauc_mrr_at_3_diff1": 0.835266, + "nauc_mrr_at_5_max": 0.524243, + "nauc_mrr_at_5_std": -0.393739, + "nauc_mrr_at_5_diff1": 0.839033, + "nauc_mrr_at_10_max": 0.519176, + "nauc_mrr_at_10_std": -0.398238, + "nauc_mrr_at_10_diff1": 0.839057, + "nauc_mrr_at_20_max": 0.518474, + "nauc_mrr_at_20_std": -0.396203, + "nauc_mrr_at_20_diff1": 0.839402, + "nauc_mrr_at_100_max": 0.518329, + "nauc_mrr_at_100_std": -0.395763, + "nauc_mrr_at_100_diff1": 0.839532, + "nauc_mrr_at_1000_max": 0.518316, + "nauc_mrr_at_1000_std": -0.395757, + "nauc_mrr_at_1000_diff1": 0.839526, + "hit_rate_at_1": 0.75334, + "hit_rate_at_3": 0.88592, + "hit_rate_at_5": 0.92292, + "hit_rate_at_10": 0.95683, + "hit_rate_at_20": 0.97842, + "hit_rate_at_100": 0.99897, + "hit_rate_at_1000": 1.0, + "main_score": 0.85149, + "hf_subset": "en", + "languages": [ + "eng-Latn" + ] + }, + { + "ndcg_at_1": 0.76915, + "ndcg_at_3": 0.85043, + "ndcg_at_5": 0.86288, + "ndcg_at_10": 0.87286, + "ndcg_at_20": 0.87834, + "ndcg_at_100": 0.88072, + "ndcg_at_1000": 0.88103, + "map_at_1": 0.74328, + "map_at_3": 0.82576, + "map_at_5": 0.83367, + "map_at_10": 0.83855, + "map_at_20": 0.84023, + "map_at_100": 0.84069, + "map_at_1000": 0.84071, + "recall_at_1": 0.74328, + "recall_at_3": 0.90824, + "recall_at_5": 0.93667, + "recall_at_10": 0.96595, + "recall_at_20": 0.98672, + "recall_at_100": 0.99796, + "recall_at_1000": 1.0, + "accuracy": 0.74328, + "precision_at_1": 0.76915, + "precision_at_3": 0.31903, + "precision_at_5": 0.19898, + "precision_at_10": 0.10317, + "precision_at_20": 0.05286, + "precision_at_100": 0.0107, + "precision_at_1000": 0.00107, + "mrr_at_1": 0.769152, + "mrr_at_3": 0.839462, + "mrr_at_5": 0.844927, + "mrr_at_10": 0.847958, + "mrr_at_20": 0.849163, + "mrr_at_100": 0.8495, + "mrr_at_1000": 0.849519, + "nauc_ndcg_at_1_max": 0.448549, + "nauc_ndcg_at_1_std": -0.294536, + "nauc_ndcg_at_1_diff1": 0.830266, + "nauc_ndcg_at_3_max": 0.478764, + "nauc_ndcg_at_3_std": -0.390955, + "nauc_ndcg_at_3_diff1": 0.792943, + "nauc_ndcg_at_5_max": 0.497253, + "nauc_ndcg_at_5_std": -0.376879, + "nauc_ndcg_at_5_diff1": 0.80746, + "nauc_ndcg_at_10_max": 0.494188, + "nauc_ndcg_at_10_std": -0.360332, + "nauc_ndcg_at_10_diff1": 0.806815, + "nauc_ndcg_at_20_max": 0.488955, + "nauc_ndcg_at_20_std": -0.34651, + "nauc_ndcg_at_20_diff1": 0.811373, + "nauc_ndcg_at_100_max": 0.483215, + "nauc_ndcg_at_100_std": -0.345071, + "nauc_ndcg_at_100_diff1": 0.810866, + "nauc_ndcg_at_1000_max": 0.481788, + "nauc_ndcg_at_1000_std": -0.347708, + "nauc_ndcg_at_1000_diff1": 0.810722, + "nauc_map_at_1_max": 0.409386, + "nauc_map_at_1_std": -0.292476, + "nauc_map_at_1_diff1": 0.809819, + "nauc_map_at_3_max": 0.464427, + "nauc_map_at_3_std": -0.366956, + "nauc_map_at_3_diff1": 0.796441, + "nauc_map_at_5_max": 0.476057, + "nauc_map_at_5_std": -0.359976, + "nauc_map_at_5_diff1": 0.80578, + "nauc_map_at_10_max": 0.475588, + "nauc_map_at_10_std": -0.352377, + "nauc_map_at_10_diff1": 0.806036, + "nauc_map_at_20_max": 0.474469, + "nauc_map_at_20_std": -0.34838, + "nauc_map_at_20_diff1": 0.807089, + "nauc_map_at_100_max": 0.473579, + "nauc_map_at_100_std": -0.348237, + "nauc_map_at_100_diff1": 0.807076, + "nauc_map_at_1000_max": 0.47351, + "nauc_map_at_1000_std": -0.348361, + "nauc_map_at_1000_diff1": 0.807068, + "nauc_recall_at_1_max": 0.409386, + "nauc_recall_at_1_std": -0.292476, + "nauc_recall_at_1_diff1": 0.809819, + "nauc_recall_at_3_max": 0.51022, + "nauc_recall_at_3_std": -0.513072, + "nauc_recall_at_3_diff1": 0.724722, + "nauc_recall_at_5_max": 0.610525, + "nauc_recall_at_5_std": -0.505, + "nauc_recall_at_5_diff1": 0.76977, + "nauc_recall_at_10_max": 0.678804, + "nauc_recall_at_10_std": -0.467999, + "nauc_recall_at_10_diff1": 0.752197, + "nauc_recall_at_20_max": 0.784851, + "nauc_recall_at_20_std": -0.26349, + "nauc_recall_at_20_diff1": 0.834151, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": 0.613542, + "nauc_recall_at_100_diff1": 0.861085, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.448549, + "nauc_precision_at_1_std": -0.294536, + "nauc_precision_at_1_diff1": 0.830266, + "nauc_precision_at_3_max": 0.353544, + "nauc_precision_at_3_std": -0.289376, + "nauc_precision_at_3_diff1": 0.308246, + "nauc_precision_at_5_max": 0.290452, + "nauc_precision_at_5_std": -0.161199, + "nauc_precision_at_5_diff1": 0.15195, + "nauc_precision_at_10_max": 0.160694, + "nauc_precision_at_10_std": 0.000404, + "nauc_precision_at_10_diff1": -0.055572, + "nauc_precision_at_20_max": 0.054343, + "nauc_precision_at_20_std": 0.132922, + "nauc_precision_at_20_diff1": -0.194624, + "nauc_precision_at_100_max": -0.02157, + "nauc_precision_at_100_std": 0.176708, + "nauc_precision_at_100_diff1": -0.280164, + "nauc_precision_at_1000_max": -0.038121, + "nauc_precision_at_1000_std": 0.16963, + "nauc_precision_at_1000_diff1": -0.298655, + "nauc_mrr_at_1_max": 0.448549, + "nauc_mrr_at_1_std": -0.294536, + "nauc_mrr_at_1_diff1": 0.830266, + "nauc_mrr_at_3_max": 0.480517, + "nauc_mrr_at_3_std": -0.354676, + "nauc_mrr_at_3_diff1": 0.818214, + "nauc_mrr_at_5_max": 0.484481, + "nauc_mrr_at_5_std": -0.344827, + "nauc_mrr_at_5_diff1": 0.821638, + "nauc_mrr_at_10_max": 0.481791, + "nauc_mrr_at_10_std": -0.341859, + "nauc_mrr_at_10_diff1": 0.820939, + "nauc_mrr_at_20_max": 0.48065, + "nauc_mrr_at_20_std": -0.339368, + "nauc_mrr_at_20_diff1": 0.822031, + "nauc_mrr_at_100_max": 0.479963, + "nauc_mrr_at_100_std": -0.339235, + "nauc_mrr_at_100_diff1": 0.821815, + "nauc_mrr_at_1000_max": 0.479891, + "nauc_mrr_at_1000_std": -0.339364, + "nauc_mrr_at_1000_diff1": 0.821808, + "hit_rate_at_1": 0.76915, + "hit_rate_at_3": 0.92339, + "hit_rate_at_5": 0.94791, + "hit_rate_at_10": 0.9714, + "hit_rate_at_20": 0.98876, + "hit_rate_at_100": 0.99796, + "hit_rate_at_1000": 1.0, + "main_score": 0.87286, + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ] + }, + { + "ndcg_at_1": 0.77013, + "ndcg_at_3": 0.83465, + "ndcg_at_5": 0.84913, + "ndcg_at_10": 0.85888, + "ndcg_at_20": 0.86642, + "ndcg_at_100": 0.87144, + "ndcg_at_1000": 0.87219, + "map_at_1": 0.74559, + "map_at_3": 0.81644, + "map_at_5": 0.8251, + "map_at_10": 0.82957, + "map_at_20": 0.8318, + "map_at_100": 0.83264, + "map_at_1000": 0.83268, + "recall_at_1": 0.74559, + "recall_at_3": 0.87853, + "recall_at_5": 0.91208, + "recall_at_10": 0.94068, + "recall_at_20": 0.96972, + "recall_at_100": 0.99514, + "recall_at_1000": 1.0, + "accuracy": 0.74559, + "precision_at_1": 0.77013, + "precision_at_3": 0.30897, + "precision_at_5": 0.19301, + "precision_at_10": 0.1, + "precision_at_20": 0.05169, + "precision_at_100": 0.01066, + "precision_at_1000": 0.00107, + "mrr_at_1": 0.770127, + "mrr_at_3": 0.825212, + "mrr_at_5": 0.832733, + "mrr_at_10": 0.836259, + "mrr_at_20": 0.838027, + "mrr_at_100": 0.838612, + "mrr_at_1000": 0.838641, + "nauc_ndcg_at_1_max": 0.457386, + "nauc_ndcg_at_1_std": -0.402328, + "nauc_ndcg_at_1_diff1": 0.801799, + "nauc_ndcg_at_3_max": 0.496264, + "nauc_ndcg_at_3_std": -0.458526, + "nauc_ndcg_at_3_diff1": 0.771678, + "nauc_ndcg_at_5_max": 0.511594, + "nauc_ndcg_at_5_std": -0.477132, + "nauc_ndcg_at_5_diff1": 0.787241, + "nauc_ndcg_at_10_max": 0.509754, + "nauc_ndcg_at_10_std": -0.470768, + "nauc_ndcg_at_10_diff1": 0.786749, + "nauc_ndcg_at_20_max": 0.507686, + "nauc_ndcg_at_20_std": -0.452155, + "nauc_ndcg_at_20_diff1": 0.784213, + "nauc_ndcg_at_100_max": 0.498451, + "nauc_ndcg_at_100_std": -0.451707, + "nauc_ndcg_at_100_diff1": 0.785184, + "nauc_ndcg_at_1000_max": 0.497116, + "nauc_ndcg_at_1000_std": -0.451035, + "nauc_ndcg_at_1000_diff1": 0.785492, + "nauc_map_at_1_max": 0.39983, + "nauc_map_at_1_std": -0.366859, + "nauc_map_at_1_diff1": 0.784902, + "nauc_map_at_3_max": 0.480235, + "nauc_map_at_3_std": -0.435238, + "nauc_map_at_3_diff1": 0.773229, + "nauc_map_at_5_max": 0.490822, + "nauc_map_at_5_std": -0.444754, + "nauc_map_at_5_diff1": 0.783158, + "nauc_map_at_10_max": 0.490075, + "nauc_map_at_10_std": -0.442873, + "nauc_map_at_10_diff1": 0.782766, + "nauc_map_at_20_max": 0.488989, + "nauc_map_at_20_std": -0.438695, + "nauc_map_at_20_diff1": 0.782032, + "nauc_map_at_100_max": 0.4876, + "nauc_map_at_100_std": -0.439027, + "nauc_map_at_100_diff1": 0.782219, + "nauc_map_at_1000_max": 0.487531, + "nauc_map_at_1000_std": -0.438946, + "nauc_map_at_1000_diff1": 0.782234, + "nauc_recall_at_1_max": 0.39983, + "nauc_recall_at_1_std": -0.366859, + "nauc_recall_at_1_diff1": 0.784902, + "nauc_recall_at_3_max": 0.533841, + "nauc_recall_at_3_std": -0.531736, + "nauc_recall_at_3_diff1": 0.741403, + "nauc_recall_at_5_max": 0.593311, + "nauc_recall_at_5_std": -0.619472, + "nauc_recall_at_5_diff1": 0.78229, + "nauc_recall_at_10_max": 0.624398, + "nauc_recall_at_10_std": -0.641045, + "nauc_recall_at_10_diff1": 0.785404, + "nauc_recall_at_20_max": 0.716925, + "nauc_recall_at_20_std": -0.505012, + "nauc_recall_at_20_diff1": 0.756975, + "nauc_recall_at_100_max": 0.717681, + "nauc_recall_at_100_std": -0.53401, + "nauc_recall_at_100_diff1": 0.726926, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.457386, + "nauc_precision_at_1_std": -0.402328, + "nauc_precision_at_1_diff1": 0.801799, + "nauc_precision_at_3_max": 0.404203, + "nauc_precision_at_3_std": -0.35401, + "nauc_precision_at_3_diff1": 0.365076, + "nauc_precision_at_5_max": 0.391988, + "nauc_precision_at_5_std": -0.344531, + "nauc_precision_at_5_diff1": 0.283497, + "nauc_precision_at_10_max": 0.294224, + "nauc_precision_at_10_std": -0.257616, + "nauc_precision_at_10_diff1": 0.124279, + "nauc_precision_at_20_max": 0.180772, + "nauc_precision_at_20_std": -0.102256, + "nauc_precision_at_20_diff1": -0.070721, + "nauc_precision_at_100_max": 0.001625, + "nauc_precision_at_100_std": 0.016702, + "nauc_precision_at_100_diff1": -0.268213, + "nauc_precision_at_1000_max": -0.038884, + "nauc_precision_at_1000_std": 0.051497, + "nauc_precision_at_1000_diff1": -0.310848, + "nauc_mrr_at_1_max": 0.457386, + "nauc_mrr_at_1_std": -0.402328, + "nauc_mrr_at_1_diff1": 0.801799, + "nauc_mrr_at_3_max": 0.497501, + "nauc_mrr_at_3_std": -0.450427, + "nauc_mrr_at_3_diff1": 0.788943, + "nauc_mrr_at_5_max": 0.499767, + "nauc_mrr_at_5_std": -0.45776, + "nauc_mrr_at_5_diff1": 0.793469, + "nauc_mrr_at_10_max": 0.498208, + "nauc_mrr_at_10_std": -0.45335, + "nauc_mrr_at_10_diff1": 0.793188, + "nauc_mrr_at_20_max": 0.49824, + "nauc_mrr_at_20_std": -0.448454, + "nauc_mrr_at_20_diff1": 0.79291, + "nauc_mrr_at_100_max": 0.497517, + "nauc_mrr_at_100_std": -0.448861, + "nauc_mrr_at_100_diff1": 0.793238, + "nauc_mrr_at_1000_max": 0.497476, + "nauc_mrr_at_1000_std": -0.44887, + "nauc_mrr_at_1000_diff1": 0.793258, + "hit_rate_at_1": 0.77013, + "hit_rate_at_3": 0.88877, + "hit_rate_at_5": 0.92055, + "hit_rate_at_10": 0.94703, + "hit_rate_at_20": 0.97352, + "hit_rate_at_100": 0.99576, + "hit_rate_at_1000": 1.0, + "main_score": 0.85888, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ] + }, + { + "ndcg_at_1": 0.68224, + "ndcg_at_3": 0.74682, + "ndcg_at_5": 0.7651, + "ndcg_at_10": 0.78495, + "ndcg_at_20": 0.79318, + "ndcg_at_100": 0.803, + "ndcg_at_1000": 0.80628, + "map_at_1": 0.66044, + "map_at_3": 0.72612, + "map_at_5": 0.73667, + "map_at_10": 0.74558, + "map_at_20": 0.74783, + "map_at_100": 0.74935, + "map_at_1000": 0.74954, + "recall_at_1": 0.66044, + "recall_at_3": 0.79318, + "recall_at_5": 0.83645, + "recall_at_10": 0.89547, + "recall_at_20": 0.92714, + "recall_at_100": 0.97767, + "recall_at_1000": 1.0, + "accuracy": 0.66044, + "precision_at_1": 0.68224, + "precision_at_3": 0.27691, + "precision_at_5": 0.17591, + "precision_at_10": 0.09502, + "precision_at_20": 0.04938, + "precision_at_100": 0.01049, + "precision_at_1000": 0.00108, + "mrr_at_1": 0.682243, + "mrr_at_3": 0.740395, + "mrr_at_5": 0.750052, + "mrr_at_10": 0.757126, + "mrr_at_20": 0.759366, + "mrr_at_100": 0.760499, + "mrr_at_1000": 0.760625, + "nauc_ndcg_at_1_max": 0.521005, + "nauc_ndcg_at_1_std": -0.323619, + "nauc_ndcg_at_1_diff1": 0.810651, + "nauc_ndcg_at_3_max": 0.521482, + "nauc_ndcg_at_3_std": -0.375137, + "nauc_ndcg_at_3_diff1": 0.773332, + "nauc_ndcg_at_5_max": 0.53592, + "nauc_ndcg_at_5_std": -0.381775, + "nauc_ndcg_at_5_diff1": 0.769312, + "nauc_ndcg_at_10_max": 0.543623, + "nauc_ndcg_at_10_std": -0.377865, + "nauc_ndcg_at_10_diff1": 0.770265, + "nauc_ndcg_at_20_max": 0.540609, + "nauc_ndcg_at_20_std": -0.385725, + "nauc_ndcg_at_20_diff1": 0.774337, + "nauc_ndcg_at_100_max": 0.536459, + "nauc_ndcg_at_100_std": -0.376381, + "nauc_ndcg_at_100_diff1": 0.776904, + "nauc_ndcg_at_1000_max": 0.53293, + "nauc_ndcg_at_1000_std": -0.372873, + "nauc_ndcg_at_1000_diff1": 0.778436, + "nauc_map_at_1_max": 0.489058, + "nauc_map_at_1_std": -0.304587, + "nauc_map_at_1_diff1": 0.783858, + "nauc_map_at_3_max": 0.513069, + "nauc_map_at_3_std": -0.363177, + "nauc_map_at_3_diff1": 0.777349, + "nauc_map_at_5_max": 0.522068, + "nauc_map_at_5_std": -0.366121, + "nauc_map_at_5_diff1": 0.776103, + "nauc_map_at_10_max": 0.525763, + "nauc_map_at_10_std": -0.364012, + "nauc_map_at_10_diff1": 0.777272, + "nauc_map_at_20_max": 0.524989, + "nauc_map_at_20_std": -0.365659, + "nauc_map_at_20_diff1": 0.778338, + "nauc_map_at_100_max": 0.524447, + "nauc_map_at_100_std": -0.365004, + "nauc_map_at_100_diff1": 0.778839, + "nauc_map_at_1000_max": 0.524264, + "nauc_map_at_1000_std": -0.364824, + "nauc_map_at_1000_diff1": 0.778887, + "nauc_recall_at_1_max": 0.489058, + "nauc_recall_at_1_std": -0.304587, + "nauc_recall_at_1_diff1": 0.783858, + "nauc_recall_at_3_max": 0.529596, + "nauc_recall_at_3_std": -0.411872, + "nauc_recall_at_3_diff1": 0.739586, + "nauc_recall_at_5_max": 0.571117, + "nauc_recall_at_5_std": -0.43885, + "nauc_recall_at_5_diff1": 0.713374, + "nauc_recall_at_10_max": 0.633112, + "nauc_recall_at_10_std": -0.453876, + "nauc_recall_at_10_diff1": 0.700387, + "nauc_recall_at_20_max": 0.636411, + "nauc_recall_at_20_std": -0.55929, + "nauc_recall_at_20_diff1": 0.704085, + "nauc_recall_at_100_max": 0.705303, + "nauc_recall_at_100_std": -0.534369, + "nauc_recall_at_100_diff1": 0.672904, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.521005, + "nauc_precision_at_1_std": -0.323619, + "nauc_precision_at_1_diff1": 0.810651, + "nauc_precision_at_3_max": 0.479072, + "nauc_precision_at_3_std": -0.392371, + "nauc_precision_at_3_diff1": 0.631677, + "nauc_precision_at_5_max": 0.485934, + "nauc_precision_at_5_std": -0.390075, + "nauc_precision_at_5_diff1": 0.547419, + "nauc_precision_at_10_max": 0.382918, + "nauc_precision_at_10_std": -0.286063, + "nauc_precision_at_10_diff1": 0.344038, + "nauc_precision_at_20_max": 0.300385, + "nauc_precision_at_20_std": -0.285211, + "nauc_precision_at_20_diff1": 0.239006, + "nauc_precision_at_100_max": 0.060466, + "nauc_precision_at_100_std": -0.071811, + "nauc_precision_at_100_diff1": -0.058784, + "nauc_precision_at_1000_max": -0.088511, + "nauc_precision_at_1000_std": 0.055825, + "nauc_precision_at_1000_diff1": -0.207257, + "nauc_mrr_at_1_max": 0.521005, + "nauc_mrr_at_1_std": -0.323619, + "nauc_mrr_at_1_diff1": 0.810651, + "nauc_mrr_at_3_max": 0.529671, + "nauc_mrr_at_3_std": -0.362433, + "nauc_mrr_at_3_diff1": 0.788376, + "nauc_mrr_at_5_max": 0.534182, + "nauc_mrr_at_5_std": -0.365181, + "nauc_mrr_at_5_diff1": 0.785654, + "nauc_mrr_at_10_max": 0.535081, + "nauc_mrr_at_10_std": -0.362773, + "nauc_mrr_at_10_diff1": 0.785299, + "nauc_mrr_at_20_max": 0.534427, + "nauc_mrr_at_20_std": -0.36458, + "nauc_mrr_at_20_diff1": 0.786486, + "nauc_mrr_at_100_max": 0.534123, + "nauc_mrr_at_100_std": -0.363571, + "nauc_mrr_at_100_diff1": 0.786877, + "nauc_mrr_at_1000_max": 0.534027, + "nauc_mrr_at_1000_std": -0.363483, + "nauc_mrr_at_1000_diff1": 0.786982, + "hit_rate_at_1": 0.68224, + "hit_rate_at_3": 0.80893, + "hit_rate_at_5": 0.85151, + "hit_rate_at_10": 0.90447, + "hit_rate_at_20": 0.93769, + "hit_rate_at_100": 0.98235, + "hit_rate_at_1000": 1.0, + "main_score": 0.78495, + "hf_subset": "es", + "languages": [ + "spa-Latn" + ] + }, + { + "ndcg_at_1": 0.85513, + "ndcg_at_3": 0.91113, + "ndcg_at_5": 0.9192, + "ndcg_at_10": 0.92543, + "ndcg_at_20": 0.92843, + "ndcg_at_100": 0.92874, + "ndcg_at_1000": 0.92882, + "map_at_1": 0.834, + "map_at_3": 0.89525, + "map_at_5": 0.90032, + "map_at_10": 0.90348, + "map_at_20": 0.90441, + "map_at_100": 0.9045, + "map_at_1000": 0.90451, + "recall_at_1": 0.834, + "recall_at_3": 0.95104, + "recall_at_5": 0.96923, + "recall_at_10": 0.98684, + "recall_at_20": 0.99824, + "recall_at_100": 0.9995, + "recall_at_1000": 1.0, + "accuracy": 0.834, + "precision_at_1": 0.85513, + "precision_at_3": 0.33233, + "precision_at_5": 0.20423, + "precision_at_10": 0.10443, + "precision_at_20": 0.05287, + "precision_at_100": 0.01059, + "precision_at_1000": 0.00106, + "mrr_at_1": 0.855131, + "mrr_at_3": 0.901241, + "mrr_at_5": 0.905114, + "mrr_at_10": 0.907151, + "mrr_at_20": 0.907865, + "mrr_at_100": 0.907913, + "mrr_at_1000": 0.907913, + "nauc_ndcg_at_1_max": 0.506691, + "nauc_ndcg_at_1_std": -0.264075, + "nauc_ndcg_at_1_diff1": 0.878249, + "nauc_ndcg_at_3_max": 0.51631, + "nauc_ndcg_at_3_std": -0.380643, + "nauc_ndcg_at_3_diff1": 0.853186, + "nauc_ndcg_at_5_max": 0.534221, + "nauc_ndcg_at_5_std": -0.347018, + "nauc_ndcg_at_5_diff1": 0.860478, + "nauc_ndcg_at_10_max": 0.527272, + "nauc_ndcg_at_10_std": -0.32808, + "nauc_ndcg_at_10_diff1": 0.86287, + "nauc_ndcg_at_20_max": 0.520239, + "nauc_ndcg_at_20_std": -0.315527, + "nauc_ndcg_at_20_diff1": 0.86279, + "nauc_ndcg_at_100_max": 0.518872, + "nauc_ndcg_at_100_std": -0.315935, + "nauc_ndcg_at_100_diff1": 0.862946, + "nauc_ndcg_at_1000_max": 0.518774, + "nauc_ndcg_at_1000_std": -0.316391, + "nauc_ndcg_at_1000_diff1": 0.863678, + "nauc_map_at_1_max": 0.454604, + "nauc_map_at_1_std": -0.272001, + "nauc_map_at_1_diff1": 0.872192, + "nauc_map_at_3_max": 0.50508, + "nauc_map_at_3_std": -0.356347, + "nauc_map_at_3_diff1": 0.856624, + "nauc_map_at_5_max": 0.514048, + "nauc_map_at_5_std": -0.337929, + "nauc_map_at_5_diff1": 0.861815, + "nauc_map_at_10_max": 0.512226, + "nauc_map_at_10_std": -0.327069, + "nauc_map_at_10_diff1": 0.862674, + "nauc_map_at_20_max": 0.510918, + "nauc_map_at_20_std": -0.323516, + "nauc_map_at_20_diff1": 0.86279, + "nauc_map_at_100_max": 0.51074, + "nauc_map_at_100_std": -0.323166, + "nauc_map_at_100_diff1": 0.862857, + "nauc_map_at_1000_max": 0.510735, + "nauc_map_at_1000_std": -0.323183, + "nauc_map_at_1000_diff1": 0.862884, + "nauc_recall_at_1_max": 0.454604, + "nauc_recall_at_1_std": -0.272001, + "nauc_recall_at_1_diff1": 0.872192, + "nauc_recall_at_3_max": 0.545266, + "nauc_recall_at_3_std": -0.594981, + "nauc_recall_at_3_diff1": 0.814196, + "nauc_recall_at_5_max": 0.661356, + "nauc_recall_at_5_std": -0.536542, + "nauc_recall_at_5_diff1": 0.829092, + "nauc_recall_at_10_max": 0.711565, + "nauc_recall_at_10_std": -0.520616, + "nauc_recall_at_10_diff1": 0.833941, + "nauc_recall_at_20_max": 0.780923, + "nauc_recall_at_20_std": -0.009183, + "nauc_recall_at_20_diff1": 0.635006, + "nauc_recall_at_100_max": 0.554225, + "nauc_recall_at_100_std": 0.122721, + "nauc_recall_at_100_diff1": 0.122721, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.506691, + "nauc_precision_at_1_std": -0.264075, + "nauc_precision_at_1_diff1": 0.878249, + "nauc_precision_at_3_max": 0.21826, + "nauc_precision_at_3_std": -0.14361, + "nauc_precision_at_3_diff1": 0.04471, + "nauc_precision_at_5_max": 0.151804, + "nauc_precision_at_5_std": 0.032231, + "nauc_precision_at_5_diff1": -0.121558, + "nauc_precision_at_10_max": 0.032931, + "nauc_precision_at_10_std": 0.184332, + "nauc_precision_at_10_diff1": -0.299957, + "nauc_precision_at_20_max": -0.046886, + "nauc_precision_at_20_std": 0.282244, + "nauc_precision_at_20_diff1": -0.421468, + "nauc_precision_at_100_max": -0.059762, + "nauc_precision_at_100_std": 0.293672, + "nauc_precision_at_100_diff1": -0.433914, + "nauc_precision_at_1000_max": -0.064069, + "nauc_precision_at_1000_std": 0.287603, + "nauc_precision_at_1000_diff1": -0.427856, + "nauc_mrr_at_1_max": 0.506691, + "nauc_mrr_at_1_std": -0.264075, + "nauc_mrr_at_1_diff1": 0.878249, + "nauc_mrr_at_3_max": 0.522926, + "nauc_mrr_at_3_std": -0.318246, + "nauc_mrr_at_3_diff1": 0.865756, + "nauc_mrr_at_5_max": 0.52987, + "nauc_mrr_at_5_std": -0.301947, + "nauc_mrr_at_5_diff1": 0.867062, + "nauc_mrr_at_10_max": 0.52592, + "nauc_mrr_at_10_std": -0.302053, + "nauc_mrr_at_10_diff1": 0.868785, + "nauc_mrr_at_20_max": 0.523964, + "nauc_mrr_at_20_std": -0.300648, + "nauc_mrr_at_20_diff1": 0.868675, + "nauc_mrr_at_100_max": 0.523708, + "nauc_mrr_at_100_std": -0.301002, + "nauc_mrr_at_100_diff1": 0.868675, + "nauc_mrr_at_1000_max": 0.523708, + "nauc_mrr_at_1000_std": -0.301002, + "nauc_mrr_at_1000_diff1": 0.868675, + "hit_rate_at_1": 0.85513, + "hit_rate_at_3": 0.95875, + "hit_rate_at_5": 0.97485, + "hit_rate_at_10": 0.98893, + "hit_rate_at_20": 0.99899, + "hit_rate_at_100": 1.0, + "hit_rate_at_1000": 1.0, + "main_score": 0.92543, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ] + }, + { + "ndcg_at_1": 0.57856, + "ndcg_at_3": 0.65173, + "ndcg_at_5": 0.6753, + "ndcg_at_10": 0.69709, + "ndcg_at_20": 0.70982, + "ndcg_at_100": 0.72547, + "ndcg_at_1000": 0.73154, + "map_at_1": 0.55324, + "map_at_3": 0.62773, + "map_at_5": 0.64165, + "map_at_10": 0.65124, + "map_at_20": 0.65487, + "map_at_100": 0.65734, + "map_at_1000": 0.65759, + "recall_at_1": 0.55324, + "recall_at_3": 0.70725, + "recall_at_5": 0.76223, + "recall_at_10": 0.8257, + "recall_at_20": 0.87444, + "recall_at_100": 0.95404, + "recall_at_1000": 1.0, + "accuracy": 0.55324, + "precision_at_1": 0.57856, + "precision_at_3": 0.25182, + "precision_at_5": 0.16379, + "precision_at_10": 0.0898, + "precision_at_20": 0.04766, + "precision_at_100": 0.01056, + "precision_at_1000": 0.00111, + "mrr_at_1": 0.578564, + "mrr_at_3": 0.642386, + "mrr_at_5": 0.654665, + "mrr_at_10": 0.663459, + "mrr_at_20": 0.666899, + "mrr_at_100": 0.668565, + "mrr_at_1000": 0.668757, + "nauc_ndcg_at_1_max": 0.372204, + "nauc_ndcg_at_1_std": -0.02767, + "nauc_ndcg_at_1_diff1": 0.754278, + "nauc_ndcg_at_3_max": 0.367908, + "nauc_ndcg_at_3_std": -0.009375, + "nauc_ndcg_at_3_diff1": 0.700478, + "nauc_ndcg_at_5_max": 0.377655, + "nauc_ndcg_at_5_std": 0.01636, + "nauc_ndcg_at_5_diff1": 0.697716, + "nauc_ndcg_at_10_max": 0.376969, + "nauc_ndcg_at_10_std": 0.018326, + "nauc_ndcg_at_10_diff1": 0.694687, + "nauc_ndcg_at_20_max": 0.379625, + "nauc_ndcg_at_20_std": 0.015957, + "nauc_ndcg_at_20_diff1": 0.699533, + "nauc_ndcg_at_100_max": 0.380505, + "nauc_ndcg_at_100_std": 0.016652, + "nauc_ndcg_at_100_diff1": 0.706768, + "nauc_ndcg_at_1000_max": 0.378762, + "nauc_ndcg_at_1000_std": 0.007787, + "nauc_ndcg_at_1000_diff1": 0.70863, + "nauc_map_at_1_max": 0.325139, + "nauc_map_at_1_std": -0.052977, + "nauc_map_at_1_diff1": 0.737335, + "nauc_map_at_3_max": 0.364389, + "nauc_map_at_3_std": -0.019891, + "nauc_map_at_3_diff1": 0.713276, + "nauc_map_at_5_max": 0.37109, + "nauc_map_at_5_std": -0.005011, + "nauc_map_at_5_diff1": 0.712779, + "nauc_map_at_10_max": 0.370847, + "nauc_map_at_10_std": -0.004598, + "nauc_map_at_10_diff1": 0.71124, + "nauc_map_at_20_max": 0.371971, + "nauc_map_at_20_std": -0.005202, + "nauc_map_at_20_diff1": 0.713036, + "nauc_map_at_100_max": 0.372143, + "nauc_map_at_100_std": -0.005168, + "nauc_map_at_100_diff1": 0.713993, + "nauc_map_at_1000_max": 0.372103, + "nauc_map_at_1000_std": -0.005455, + "nauc_map_at_1000_diff1": 0.714076, + "nauc_recall_at_1_max": 0.325139, + "nauc_recall_at_1_std": -0.052977, + "nauc_recall_at_1_diff1": 0.737335, + "nauc_recall_at_3_max": 0.358563, + "nauc_recall_at_3_std": 0.008264, + "nauc_recall_at_3_diff1": 0.644893, + "nauc_recall_at_5_max": 0.383863, + "nauc_recall_at_5_std": 0.08487, + "nauc_recall_at_5_diff1": 0.627091, + "nauc_recall_at_10_max": 0.378897, + "nauc_recall_at_10_std": 0.116858, + "nauc_recall_at_10_diff1": 0.596062, + "nauc_recall_at_20_max": 0.394787, + "nauc_recall_at_20_std": 0.131308, + "nauc_recall_at_20_diff1": 0.598163, + "nauc_recall_at_100_max": 0.44838, + "nauc_recall_at_100_std": 0.340335, + "nauc_recall_at_100_diff1": 0.636752, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.372204, + "nauc_precision_at_1_std": -0.02767, + "nauc_precision_at_1_diff1": 0.754278, + "nauc_precision_at_3_max": 0.386692, + "nauc_precision_at_3_std": 0.055841, + "nauc_precision_at_3_diff1": 0.531504, + "nauc_precision_at_5_max": 0.383282, + "nauc_precision_at_5_std": 0.123402, + "nauc_precision_at_5_diff1": 0.453114, + "nauc_precision_at_10_max": 0.340546, + "nauc_precision_at_10_std": 0.137408, + "nauc_precision_at_10_diff1": 0.316071, + "nauc_precision_at_20_max": 0.33166, + "nauc_precision_at_20_std": 0.152996, + "nauc_precision_at_20_diff1": 0.240889, + "nauc_precision_at_100_max": 0.177604, + "nauc_precision_at_100_std": 0.147208, + "nauc_precision_at_100_diff1": -0.019757, + "nauc_precision_at_1000_max": 0.068166, + "nauc_precision_at_1000_std": 0.063012, + "nauc_precision_at_1000_diff1": -0.198644, + "nauc_mrr_at_1_max": 0.372204, + "nauc_mrr_at_1_std": -0.02767, + "nauc_mrr_at_1_diff1": 0.754278, + "nauc_mrr_at_3_max": 0.37918, + "nauc_mrr_at_3_std": -0.003364, + "nauc_mrr_at_3_diff1": 0.718395, + "nauc_mrr_at_5_max": 0.38236, + "nauc_mrr_at_5_std": 0.00799, + "nauc_mrr_at_5_diff1": 0.716664, + "nauc_mrr_at_10_max": 0.381917, + "nauc_mrr_at_10_std": 0.007719, + "nauc_mrr_at_10_diff1": 0.716646, + "nauc_mrr_at_20_max": 0.382119, + "nauc_mrr_at_20_std": 0.006359, + "nauc_mrr_at_20_diff1": 0.717674, + "nauc_mrr_at_100_max": 0.38212, + "nauc_mrr_at_100_std": 0.006276, + "nauc_mrr_at_100_diff1": 0.718577, + "nauc_mrr_at_1000_max": 0.382152, + "nauc_mrr_at_1000_std": 0.006099, + "nauc_mrr_at_1000_diff1": 0.71867, + "hit_rate_at_1": 0.57856, + "hit_rate_at_3": 0.72425, + "hit_rate_at_5": 0.77732, + "hit_rate_at_10": 0.84183, + "hit_rate_at_20": 0.89074, + "hit_rate_at_100": 0.95942, + "hit_rate_at_1000": 1.0, + "main_score": 0.69709, + "hf_subset": "sw", + "languages": [ + "swa-Latn" + ] + }, + { + "ndcg_at_1": 0.84333, + "ndcg_at_3": 0.89374, + "ndcg_at_5": 0.90696, + "ndcg_at_10": 0.91389, + "ndcg_at_20": 0.91609, + "ndcg_at_100": 0.91824, + "ndcg_at_1000": 0.91824, + "map_at_1": 0.82185, + "map_at_3": 0.87944, + "map_at_5": 0.88744, + "map_at_10": 0.8905, + "map_at_20": 0.89116, + "map_at_100": 0.89153, + "map_at_1000": 0.89153, + "recall_at_1": 0.82185, + "recall_at_3": 0.92944, + "recall_at_5": 0.96056, + "recall_at_10": 0.98093, + "recall_at_20": 0.98907, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "accuracy": 0.82185, + "precision_at_1": 0.84333, + "precision_at_3": 0.32407, + "precision_at_5": 0.20133, + "precision_at_10": 0.103, + "precision_at_20": 0.05206, + "precision_at_100": 0.01053, + "precision_at_1000": 0.00105, + "mrr_at_1": 0.843333, + "mrr_at_3": 0.885, + "mrr_at_5": 0.891444, + "mrr_at_10": 0.894253, + "mrr_at_20": 0.894785, + "mrr_at_100": 0.895066, + "mrr_at_1000": 0.895066, + "nauc_ndcg_at_1_max": 0.450688, + "nauc_ndcg_at_1_std": -0.32737, + "nauc_ndcg_at_1_diff1": 0.825795, + "nauc_ndcg_at_3_max": 0.417674, + "nauc_ndcg_at_3_std": -0.406546, + "nauc_ndcg_at_3_diff1": 0.80247, + "nauc_ndcg_at_5_max": 0.434467, + "nauc_ndcg_at_5_std": -0.385633, + "nauc_ndcg_at_5_diff1": 0.81014, + "nauc_ndcg_at_10_max": 0.449596, + "nauc_ndcg_at_10_std": -0.357035, + "nauc_ndcg_at_10_diff1": 0.818021, + "nauc_ndcg_at_20_max": 0.449261, + "nauc_ndcg_at_20_std": -0.356496, + "nauc_ndcg_at_20_diff1": 0.816638, + "nauc_ndcg_at_100_max": 0.44234, + "nauc_ndcg_at_100_std": -0.365146, + "nauc_ndcg_at_100_diff1": 0.814709, + "nauc_ndcg_at_1000_max": 0.44234, + "nauc_ndcg_at_1000_std": -0.365146, + "nauc_ndcg_at_1000_diff1": 0.814709, + "nauc_map_at_1_max": 0.375095, + "nauc_map_at_1_std": -0.31886, + "nauc_map_at_1_diff1": 0.807197, + "nauc_map_at_3_max": 0.409057, + "nauc_map_at_3_std": -0.394904, + "nauc_map_at_3_diff1": 0.804642, + "nauc_map_at_5_max": 0.423077, + "nauc_map_at_5_std": -0.382079, + "nauc_map_at_5_diff1": 0.808931, + "nauc_map_at_10_max": 0.429828, + "nauc_map_at_10_std": -0.371016, + "nauc_map_at_10_diff1": 0.813017, + "nauc_map_at_20_max": 0.429451, + "nauc_map_at_20_std": -0.370784, + "nauc_map_at_20_diff1": 0.812921, + "nauc_map_at_100_max": 0.428785, + "nauc_map_at_100_std": -0.372004, + "nauc_map_at_100_diff1": 0.812555, + "nauc_map_at_1000_max": 0.428785, + "nauc_map_at_1000_std": -0.372004, + "nauc_map_at_1000_diff1": 0.812555, + "nauc_recall_at_1_max": 0.375095, + "nauc_recall_at_1_std": -0.31886, + "nauc_recall_at_1_diff1": 0.807197, + "nauc_recall_at_3_max": 0.394024, + "nauc_recall_at_3_std": -0.490935, + "nauc_recall_at_3_diff1": 0.777089, + "nauc_recall_at_5_max": 0.445227, + "nauc_recall_at_5_std": -0.444135, + "nauc_recall_at_5_diff1": 0.799238, + "nauc_recall_at_10_max": 0.625067, + "nauc_recall_at_10_std": -0.155997, + "nauc_recall_at_10_diff1": 0.874471, + "nauc_recall_at_20_max": 0.735468, + "nauc_recall_at_20_std": -0.036074, + "nauc_recall_at_20_diff1": 0.875374, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.450688, + "nauc_precision_at_1_std": -0.32737, + "nauc_precision_at_1_diff1": 0.825795, + "nauc_precision_at_3_max": 0.246721, + "nauc_precision_at_3_std": -0.227305, + "nauc_precision_at_3_diff1": 0.223615, + "nauc_precision_at_5_max": 0.214436, + "nauc_precision_at_5_std": -0.083233, + "nauc_precision_at_5_diff1": 0.032793, + "nauc_precision_at_10_max": 0.18731, + "nauc_precision_at_10_std": 0.073756, + "nauc_precision_at_10_diff1": -0.119368, + "nauc_precision_at_20_max": 0.147677, + "nauc_precision_at_20_std": 0.139608, + "nauc_precision_at_20_diff1": -0.218729, + "nauc_precision_at_100_max": 0.071975, + "nauc_precision_at_100_std": 0.157734, + "nauc_precision_at_100_diff1": -0.35022, + "nauc_precision_at_1000_max": 0.071975, + "nauc_precision_at_1000_std": 0.157734, + "nauc_precision_at_1000_diff1": -0.35022, + "nauc_mrr_at_1_max": 0.450688, + "nauc_mrr_at_1_std": -0.32737, + "nauc_mrr_at_1_diff1": 0.825795, + "nauc_mrr_at_3_max": 0.453448, + "nauc_mrr_at_3_std": -0.367914, + "nauc_mrr_at_3_diff1": 0.81392, + "nauc_mrr_at_5_max": 0.453103, + "nauc_mrr_at_5_std": -0.361803, + "nauc_mrr_at_5_diff1": 0.81802, + "nauc_mrr_at_10_max": 0.457177, + "nauc_mrr_at_10_std": -0.352987, + "nauc_mrr_at_10_diff1": 0.818899, + "nauc_mrr_at_20_max": 0.457572, + "nauc_mrr_at_20_std": -0.352815, + "nauc_mrr_at_20_diff1": 0.818587, + "nauc_mrr_at_100_max": 0.456701, + "nauc_mrr_at_100_std": -0.353861, + "nauc_mrr_at_100_diff1": 0.818432, + "nauc_mrr_at_1000_max": 0.456701, + "nauc_mrr_at_1000_std": -0.353861, + "nauc_mrr_at_1000_diff1": 0.818432, + "hit_rate_at_1": 0.84333, + "hit_rate_at_3": 0.93444, + "hit_rate_at_5": 0.96222, + "hit_rate_at_10": 0.98222, + "hit_rate_at_20": 0.99, + "hit_rate_at_100": 1.0, + "hit_rate_at_1000": 1.0, + "main_score": 0.91389, + "hf_subset": "hi", + "languages": [ + "hin-Deva" + ] + }, + { + "ndcg_at_1": 0.76915, + "ndcg_at_3": 0.84401, + "ndcg_at_5": 0.86077, + "ndcg_at_10": 0.86998, + "ndcg_at_20": 0.87565, + "ndcg_at_100": 0.87889, + "ndcg_at_1000": 0.87896, + "map_at_1": 0.74089, + "map_at_3": 0.82176, + "map_at_5": 0.83192, + "map_at_10": 0.83644, + "map_at_20": 0.83816, + "map_at_100": 0.83873, + "map_at_1000": 0.83874, + "recall_at_1": 0.74089, + "recall_at_3": 0.89683, + "recall_at_5": 0.93548, + "recall_at_10": 0.96204, + "recall_at_20": 0.98383, + "recall_at_100": 0.99966, + "recall_at_1000": 1.0, + "accuracy": 0.74089, + "precision_at_1": 0.76915, + "precision_at_3": 0.32108, + "precision_at_5": 0.20204, + "precision_at_10": 0.1046, + "precision_at_20": 0.05352, + "precision_at_100": 0.01091, + "precision_at_1000": 0.00109, + "mrr_at_1": 0.769152, + "mrr_at_3": 0.832652, + "mrr_at_5": 0.84113, + "mrr_at_10": 0.843789, + "mrr_at_20": 0.845135, + "mrr_at_100": 0.845531, + "mrr_at_1000": 0.845531, + "nauc_ndcg_at_1_max": 0.422452, + "nauc_ndcg_at_1_std": -0.289551, + "nauc_ndcg_at_1_diff1": 0.818072, + "nauc_ndcg_at_3_max": 0.455249, + "nauc_ndcg_at_3_std": -0.323442, + "nauc_ndcg_at_3_diff1": 0.794734, + "nauc_ndcg_at_5_max": 0.450068, + "nauc_ndcg_at_5_std": -0.351091, + "nauc_ndcg_at_5_diff1": 0.798124, + "nauc_ndcg_at_10_max": 0.454418, + "nauc_ndcg_at_10_std": -0.335027, + "nauc_ndcg_at_10_diff1": 0.805401, + "nauc_ndcg_at_20_max": 0.453108, + "nauc_ndcg_at_20_std": -0.32218, + "nauc_ndcg_at_20_diff1": 0.805835, + "nauc_ndcg_at_100_max": 0.447251, + "nauc_ndcg_at_100_std": -0.320838, + "nauc_ndcg_at_100_diff1": 0.806093, + "nauc_ndcg_at_1000_max": 0.446908, + "nauc_ndcg_at_1000_std": -0.321659, + "nauc_ndcg_at_1000_diff1": 0.806054, + "nauc_map_at_1_max": 0.380543, + "nauc_map_at_1_std": -0.290077, + "nauc_map_at_1_diff1": 0.808353, + "nauc_map_at_3_max": 0.438764, + "nauc_map_at_3_std": -0.320917, + "nauc_map_at_3_diff1": 0.802591, + "nauc_map_at_5_max": 0.436354, + "nauc_map_at_5_std": -0.334547, + "nauc_map_at_5_diff1": 0.803528, + "nauc_map_at_10_max": 0.439067, + "nauc_map_at_10_std": -0.328473, + "nauc_map_at_10_diff1": 0.806498, + "nauc_map_at_20_max": 0.438111, + "nauc_map_at_20_std": -0.325749, + "nauc_map_at_20_diff1": 0.806676, + "nauc_map_at_100_max": 0.437739, + "nauc_map_at_100_std": -0.324873, + "nauc_map_at_100_diff1": 0.806777, + "nauc_map_at_1000_max": 0.437706, + "nauc_map_at_1000_std": -0.324949, + "nauc_map_at_1000_diff1": 0.806774, + "nauc_recall_at_1_max": 0.380543, + "nauc_recall_at_1_std": -0.290077, + "nauc_recall_at_1_diff1": 0.808353, + "nauc_recall_at_3_max": 0.501407, + "nauc_recall_at_3_std": -0.359734, + "nauc_recall_at_3_diff1": 0.757691, + "nauc_recall_at_5_max": 0.491346, + "nauc_recall_at_5_std": -0.517912, + "nauc_recall_at_5_diff1": 0.749026, + "nauc_recall_at_10_max": 0.55667, + "nauc_recall_at_10_std": -0.474907, + "nauc_recall_at_10_diff1": 0.790176, + "nauc_recall_at_20_max": 0.689283, + "nauc_recall_at_20_std": -0.297857, + "nauc_recall_at_20_diff1": 0.793112, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": 1.0, + "nauc_recall_at_100_diff1": 0.869266, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.422452, + "nauc_precision_at_1_std": -0.289551, + "nauc_precision_at_1_diff1": 0.818072, + "nauc_precision_at_3_max": 0.268632, + "nauc_precision_at_3_std": -0.159015, + "nauc_precision_at_3_diff1": 0.221252, + "nauc_precision_at_5_max": 0.173053, + "nauc_precision_at_5_std": -0.142687, + "nauc_precision_at_5_diff1": 0.034428, + "nauc_precision_at_10_max": 0.094148, + "nauc_precision_at_10_std": -0.01528, + "nauc_precision_at_10_diff1": -0.112204, + "nauc_precision_at_20_max": 0.027053, + "nauc_precision_at_20_std": 0.073639, + "nauc_precision_at_20_diff1": -0.238843, + "nauc_precision_at_100_max": -0.052697, + "nauc_precision_at_100_std": 0.123933, + "nauc_precision_at_100_diff1": -0.339517, + "nauc_precision_at_1000_max": -0.058502, + "nauc_precision_at_1000_std": 0.116165, + "nauc_precision_at_1000_diff1": -0.3413, + "nauc_mrr_at_1_max": 0.422452, + "nauc_mrr_at_1_std": -0.289551, + "nauc_mrr_at_1_diff1": 0.818072, + "nauc_mrr_at_3_max": 0.455346, + "nauc_mrr_at_3_std": -0.299102, + "nauc_mrr_at_3_diff1": 0.79999, + "nauc_mrr_at_5_max": 0.452233, + "nauc_mrr_at_5_std": -0.308607, + "nauc_mrr_at_5_diff1": 0.804149, + "nauc_mrr_at_10_max": 0.451085, + "nauc_mrr_at_10_std": -0.305833, + "nauc_mrr_at_10_diff1": 0.806506, + "nauc_mrr_at_20_max": 0.450785, + "nauc_mrr_at_20_std": -0.303054, + "nauc_mrr_at_20_diff1": 0.80644, + "nauc_mrr_at_100_max": 0.450014, + "nauc_mrr_at_100_std": -0.303307, + "nauc_mrr_at_100_diff1": 0.80645, + "nauc_mrr_at_1000_max": 0.450014, + "nauc_mrr_at_1000_std": -0.303307, + "nauc_mrr_at_1000_diff1": 0.80645, + "hit_rate_at_1": 0.76915, + "hit_rate_at_3": 0.91011, + "hit_rate_at_5": 0.94688, + "hit_rate_at_10": 0.96731, + "hit_rate_at_20": 0.98672, + "hit_rate_at_100": 1.0, + "hit_rate_at_1000": 1.0, + "main_score": 0.86998, + "hf_subset": "ur", + "languages": [ + "urd-Arab" + ] + } + ] + }, + "evaluation_time": 574.7596521377563, + "kg_co2_emissions": null, + "date": 1777841967.356765 +} \ No newline at end of file diff --git a/results/nvidia__llama-nemotron-embed-1b-v2/b4caa8456edd360b3b4e938d94ed4398dd437fad/experiments/trust_remote_code_True/LexRetrieval.v1.json b/results/nvidia__llama-nemotron-embed-1b-v2/b4caa8456edd360b3b4e938d94ed4398dd437fad/experiments/trust_remote_code_True/LexRetrieval.v1.json new file mode 100644 index 0000000000..914badcbac --- /dev/null +++ b/results/nvidia__llama-nemotron-embed-1b-v2/b4caa8456edd360b3b4e938d94ed4398dd437fad/experiments/trust_remote_code_True/LexRetrieval.v1.json @@ -0,0 +1,167 @@ +{ + "dataset_revision": "b5b22dee3a30bb32ac579c14afe9eb152e8c70c7", + "task_name": "LexRetrieval.v1", + "mteb_version": "2.12.38", + "scores": { + "test": [ + { + "ndcg_at_1": 0.96381, + "ndcg_at_3": 0.97598, + "ndcg_at_5": 0.9774, + "ndcg_at_10": 0.97883, + "ndcg_at_20": 0.97949, + "ndcg_at_100": 0.98026, + "ndcg_at_1000": 0.98045, + "map_at_1": 0.96381, + "map_at_3": 0.97322, + "map_at_5": 0.97401, + "map_at_10": 0.97461, + "map_at_20": 0.97479, + "map_at_100": 0.97489, + "map_at_1000": 0.9749, + "recall_at_1": 0.96381, + "recall_at_3": 0.98388, + "recall_at_5": 0.9873, + "recall_at_10": 0.99167, + "recall_at_20": 0.99434, + "recall_at_100": 0.99851, + "recall_at_1000": 1.0, + "accuracy": 0.96381, + "precision_at_1": 0.96381, + "precision_at_3": 0.32796, + "precision_at_5": 0.19746, + "precision_at_10": 0.09917, + "precision_at_20": 0.04972, + "precision_at_100": 0.00999, + "precision_at_1000": 0.001, + "mrr_at_1": 0.963809, + "mrr_at_3": 0.973222, + "mrr_at_5": 0.974012, + "mrr_at_10": 0.974609, + "mrr_at_20": 0.974788, + "mrr_at_100": 0.974892, + "mrr_at_1000": 0.9749, + "nauc_ndcg_at_1_max": 0.783811, + "nauc_ndcg_at_1_std": -0.010488, + "nauc_ndcg_at_1_diff1": 0.970042, + "nauc_ndcg_at_3_max": 0.866073, + "nauc_ndcg_at_3_std": 0.025247, + "nauc_ndcg_at_3_diff1": 0.976589, + "nauc_ndcg_at_5_max": 0.859531, + "nauc_ndcg_at_5_std": 0.012366, + "nauc_ndcg_at_5_diff1": 0.976695, + "nauc_ndcg_at_10_max": 0.851145, + "nauc_ndcg_at_10_std": -0.003113, + "nauc_ndcg_at_10_diff1": 0.975923, + "nauc_ndcg_at_20_max": 0.846796, + "nauc_ndcg_at_20_std": -0.004584, + "nauc_ndcg_at_20_diff1": 0.975475, + "nauc_ndcg_at_100_max": 0.842218, + "nauc_ndcg_at_100_std": 0.006623, + "nauc_ndcg_at_100_diff1": 0.974523, + "nauc_ndcg_at_1000_max": 0.840938, + "nauc_ndcg_at_1000_std": 0.004022, + "nauc_ndcg_at_1000_diff1": 0.974637, + "nauc_map_at_1_max": 0.783811, + "nauc_map_at_1_std": -0.010488, + "nauc_map_at_1_diff1": 0.970042, + "nauc_map_at_3_max": 0.842996, + "nauc_map_at_3_std": 0.0155, + "nauc_map_at_3_diff1": 0.974535, + "nauc_map_at_5_max": 0.839112, + "nauc_map_at_5_std": 0.008982, + "nauc_map_at_5_diff1": 0.974555, + "nauc_map_at_10_max": 0.835703, + "nauc_map_at_10_std": 0.003324, + "nauc_map_at_10_diff1": 0.974268, + "nauc_map_at_20_max": 0.834638, + "nauc_map_at_20_std": 0.003092, + "nauc_map_at_20_diff1": 0.974156, + "nauc_map_at_100_max": 0.834067, + "nauc_map_at_100_std": 0.004328, + "nauc_map_at_100_diff1": 0.974049, + "nauc_map_at_1000_max": 0.834023, + "nauc_map_at_1000_std": 0.00426, + "nauc_map_at_1000_diff1": 0.974048, + "nauc_recall_at_1_max": 0.783811, + "nauc_recall_at_1_std": -0.010488, + "nauc_recall_at_1_diff1": 0.970042, + "nauc_recall_at_3_max": 0.974137, + "nauc_recall_at_3_std": 0.070671, + "nauc_recall_at_3_diff1": 0.986381, + "nauc_recall_at_5_max": 0.975147, + "nauc_recall_at_5_std": 0.027285, + "nauc_recall_at_5_diff1": 0.989211, + "nauc_recall_at_10_max": 0.970672, + "nauc_recall_at_10_std": -0.083025, + "nauc_recall_at_10_diff1": 0.989256, + "nauc_recall_at_20_max": 0.964237, + "nauc_recall_at_20_std": -0.143953, + "nauc_recall_at_20_diff1": 0.989121, + "nauc_recall_at_100_max": 0.970822, + "nauc_recall_at_100_std": 0.277907, + "nauc_recall_at_100_diff1": 0.958817, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.783811, + "nauc_precision_at_1_std": -0.010488, + "nauc_precision_at_1_diff1": 0.970042, + "nauc_precision_at_3_max": 0.974137, + "nauc_precision_at_3_std": 0.070671, + "nauc_precision_at_3_diff1": 0.986381, + "nauc_precision_at_5_max": 0.975147, + "nauc_precision_at_5_std": 0.027285, + "nauc_precision_at_5_diff1": 0.989211, + "nauc_precision_at_10_max": 0.970672, + "nauc_precision_at_10_std": -0.083025, + "nauc_precision_at_10_diff1": 0.989256, + "nauc_precision_at_20_max": 0.964237, + "nauc_precision_at_20_std": -0.143953, + "nauc_precision_at_20_diff1": 0.989121, + "nauc_precision_at_100_max": 0.970822, + "nauc_precision_at_100_std": 0.277907, + "nauc_precision_at_100_diff1": 0.958817, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_mrr_at_1_max": 0.783811, + "nauc_mrr_at_1_std": -0.010488, + "nauc_mrr_at_1_diff1": 0.970042, + "nauc_mrr_at_3_max": 0.842996, + "nauc_mrr_at_3_std": 0.0155, + "nauc_mrr_at_3_diff1": 0.974535, + "nauc_mrr_at_5_max": 0.839112, + "nauc_mrr_at_5_std": 0.008982, + "nauc_mrr_at_5_diff1": 0.974555, + "nauc_mrr_at_10_max": 0.835703, + "nauc_mrr_at_10_std": 0.003324, + "nauc_mrr_at_10_diff1": 0.974268, + "nauc_mrr_at_20_max": 0.834638, + "nauc_mrr_at_20_std": 0.003092, + "nauc_mrr_at_20_diff1": 0.974156, + "nauc_mrr_at_100_max": 0.834067, + "nauc_mrr_at_100_std": 0.004328, + "nauc_mrr_at_100_diff1": 0.974049, + "nauc_mrr_at_1000_max": 0.834023, + "nauc_mrr_at_1000_std": 0.00426, + "nauc_mrr_at_1000_diff1": 0.974048, + "hit_rate_at_1": 0.96381, + "hit_rate_at_3": 0.98388, + "hit_rate_at_5": 0.9873, + "hit_rate_at_10": 0.99167, + "hit_rate_at_20": 0.99434, + "hit_rate_at_100": 0.99851, + "hit_rate_at_1000": 1.0, + "main_score": 0.97883, + "hf_subset": "default", + "languages": [ + "dan-Latn" + ] + } + ] + }, + "evaluation_time": 403.6956853866577, + "kg_co2_emissions": null, + "date": 1777841372.289786 +} \ No newline at end of file diff --git a/results/nvidia__llama-nemotron-embed-1b-v2/b4caa8456edd360b3b4e938d94ed4398dd437fad/experiments/trust_remote_code_True/PublicNewsRetrieval.json b/results/nvidia__llama-nemotron-embed-1b-v2/b4caa8456edd360b3b4e938d94ed4398dd437fad/experiments/trust_remote_code_True/PublicNewsRetrieval.json new file mode 100644 index 0000000000..928052460c --- /dev/null +++ b/results/nvidia__llama-nemotron-embed-1b-v2/b4caa8456edd360b3b4e938d94ed4398dd437fad/experiments/trust_remote_code_True/PublicNewsRetrieval.json @@ -0,0 +1,322 @@ +{ + "dataset_revision": "82dad8c469646feae9631d69ba92bebfc0a586e9", + "task_name": "PublicNewsRetrieval", + "mteb_version": "2.12.37", + "scores": { + "test": [ + { + "ndcg_at_1": 0.79692, + "ndcg_at_3": 0.84926, + "ndcg_at_5": 0.86121, + "ndcg_at_10": 0.87095, + "ndcg_at_20": 0.87604, + "ndcg_at_100": 0.88086, + "ndcg_at_1000": 0.88235, + "map_at_1": 0.78569, + "map_at_3": 0.83441, + "map_at_5": 0.8416, + "map_at_10": 0.84593, + "map_at_20": 0.84741, + "map_at_100": 0.8481, + "map_at_1000": 0.84817, + "recall_at_1": 0.78569, + "recall_at_3": 0.88539, + "recall_at_5": 0.91393, + "recall_at_10": 0.94253, + "recall_at_20": 0.96202, + "recall_at_100": 0.98766, + "recall_at_1000": 0.99866, + "accuracy": 0.78569, + "precision_at_1": 0.79692, + "precision_at_3": 0.30317, + "precision_at_5": 0.18887, + "precision_at_10": 0.09786, + "precision_at_20": 0.05013, + "precision_at_100": 0.01031, + "precision_at_1000": 0.00104, + "mrr_at_1": 0.796917, + "mrr_at_3": 0.840706, + "mrr_at_5": 0.846437, + "mrr_at_10": 0.850337, + "mrr_at_20": 0.851542, + "mrr_at_100": 0.852135, + "mrr_at_1000": 0.852178, + "nauc_ndcg_at_1_max": 0.509929, + "nauc_ndcg_at_1_std": -0.36153, + "nauc_ndcg_at_1_diff1": 0.881341, + "nauc_ndcg_at_3_max": 0.559952, + "nauc_ndcg_at_3_std": -0.338943, + "nauc_ndcg_at_3_diff1": 0.857358, + "nauc_ndcg_at_5_max": 0.566239, + "nauc_ndcg_at_5_std": -0.335531, + "nauc_ndcg_at_5_diff1": 0.858818, + "nauc_ndcg_at_10_max": 0.571754, + "nauc_ndcg_at_10_std": -0.325553, + "nauc_ndcg_at_10_diff1": 0.86036, + "nauc_ndcg_at_20_max": 0.571747, + "nauc_ndcg_at_20_std": -0.324243, + "nauc_ndcg_at_20_diff1": 0.863029, + "nauc_ndcg_at_100_max": 0.56343, + "nauc_ndcg_at_100_std": -0.324489, + "nauc_ndcg_at_100_diff1": 0.863522, + "nauc_ndcg_at_1000_max": 0.561289, + "nauc_ndcg_at_1000_std": -0.330309, + "nauc_ndcg_at_1000_diff1": 0.865035, + "nauc_map_at_1_max": 0.483224, + "nauc_map_at_1_std": -0.371833, + "nauc_map_at_1_diff1": 0.879063, + "nauc_map_at_3_max": 0.544176, + "nauc_map_at_3_std": -0.351567, + "nauc_map_at_3_diff1": 0.863971, + "nauc_map_at_5_max": 0.54693, + "nauc_map_at_5_std": -0.348547, + "nauc_map_at_5_diff1": 0.864816, + "nauc_map_at_10_max": 0.549564, + "nauc_map_at_10_std": -0.346506, + "nauc_map_at_10_diff1": 0.865818, + "nauc_map_at_20_max": 0.549785, + "nauc_map_at_20_std": -0.345723, + "nauc_map_at_20_diff1": 0.866432, + "nauc_map_at_100_max": 0.548803, + "nauc_map_at_100_std": -0.345496, + "nauc_map_at_100_diff1": 0.866512, + "nauc_map_at_1000_max": 0.548766, + "nauc_map_at_1000_std": -0.345671, + "nauc_map_at_1000_diff1": 0.866593, + "nauc_recall_at_1_max": 0.483224, + "nauc_recall_at_1_std": -0.371833, + "nauc_recall_at_1_diff1": 0.879063, + "nauc_recall_at_3_max": 0.606385, + "nauc_recall_at_3_std": -0.31063, + "nauc_recall_at_3_diff1": 0.826096, + "nauc_recall_at_5_max": 0.639544, + "nauc_recall_at_5_std": -0.290187, + "nauc_recall_at_5_diff1": 0.816404, + "nauc_recall_at_10_max": 0.699163, + "nauc_recall_at_10_std": -0.203095, + "nauc_recall_at_10_diff1": 0.806745, + "nauc_recall_at_20_max": 0.76088, + "nauc_recall_at_20_std": -0.134874, + "nauc_recall_at_20_diff1": 0.815965, + "nauc_recall_at_100_max": 0.750146, + "nauc_recall_at_100_std": 0.182003, + "nauc_recall_at_100_diff1": 0.757697, + "nauc_recall_at_1000_max": 0.93465, + "nauc_recall_at_1000_std": 0.487867, + "nauc_recall_at_1000_diff1": 0.897898, + "nauc_precision_at_1_max": 0.509929, + "nauc_precision_at_1_std": -0.36153, + "nauc_precision_at_1_diff1": 0.881341, + "nauc_precision_at_3_max": 0.537094, + "nauc_precision_at_3_std": -0.171073, + "nauc_precision_at_3_diff1": 0.58273, + "nauc_precision_at_5_max": 0.460153, + "nauc_precision_at_5_std": -0.080075, + "nauc_precision_at_5_diff1": 0.414105, + "nauc_precision_at_10_max": 0.406632, + "nauc_precision_at_10_std": 0.065243, + "nauc_precision_at_10_diff1": 0.223719, + "nauc_precision_at_20_max": 0.307152, + "nauc_precision_at_20_std": 0.181569, + "nauc_precision_at_20_diff1": 0.042039, + "nauc_precision_at_100_max": 0.113371, + "nauc_precision_at_100_std": 0.355515, + "nauc_precision_at_100_diff1": -0.234803, + "nauc_precision_at_1000_max": -0.005334, + "nauc_precision_at_1000_std": 0.35968, + "nauc_precision_at_1000_diff1": -0.363059, + "nauc_mrr_at_1_max": 0.509929, + "nauc_mrr_at_1_std": -0.36153, + "nauc_mrr_at_1_diff1": 0.881341, + "nauc_mrr_at_3_max": 0.551475, + "nauc_mrr_at_3_std": -0.33681, + "nauc_mrr_at_3_diff1": 0.865007, + "nauc_mrr_at_5_max": 0.554148, + "nauc_mrr_at_5_std": -0.336054, + "nauc_mrr_at_5_diff1": 0.866748, + "nauc_mrr_at_10_max": 0.554468, + "nauc_mrr_at_10_std": -0.332283, + "nauc_mrr_at_10_diff1": 0.866942, + "nauc_mrr_at_20_max": 0.55371, + "nauc_mrr_at_20_std": -0.333473, + "nauc_mrr_at_20_diff1": 0.867731, + "nauc_mrr_at_100_max": 0.552759, + "nauc_mrr_at_100_std": -0.333965, + "nauc_mrr_at_100_diff1": 0.867708, + "nauc_mrr_at_1000_max": 0.552684, + "nauc_mrr_at_1000_std": -0.334166, + "nauc_mrr_at_1000_diff1": 0.867736, + "hit_rate_at_1": 0.79692, + "hit_rate_at_3": 0.89343, + "hit_rate_at_5": 0.91957, + "hit_rate_at_10": 0.94772, + "hit_rate_at_20": 0.96582, + "hit_rate_at_100": 0.98995, + "hit_rate_at_1000": 0.99933, + "main_score": 0.87095, + "hf_subset": "pl", + "languages": [ + "pol-Latn" + ] + }, + { + "ndcg_at_1": 0.75414, + "ndcg_at_3": 0.8231, + "ndcg_at_5": 0.84115, + "ndcg_at_10": 0.85282, + "ndcg_at_20": 0.85838, + "ndcg_at_100": 0.86273, + "ndcg_at_1000": 0.8639, + "map_at_1": 0.73349, + "map_at_3": 0.80294, + "map_at_5": 0.81408, + "map_at_10": 0.81933, + "map_at_20": 0.82093, + "map_at_100": 0.82162, + "map_at_1000": 0.82168, + "recall_at_1": 0.73349, + "recall_at_3": 0.87273, + "recall_at_5": 0.9145, + "recall_at_10": 0.94894, + "recall_at_20": 0.97037, + "recall_at_100": 0.99196, + "recall_at_1000": 1.0, + "accuracy": 0.73349, + "precision_at_1": 0.75414, + "precision_at_3": 0.30575, + "precision_at_5": 0.19338, + "precision_at_10": 0.10071, + "precision_at_20": 0.05158, + "precision_at_100": 0.0106, + "precision_at_1000": 0.00107, + "mrr_at_1": 0.754137, + "mrr_at_3": 0.811426, + "mrr_at_5": 0.820173, + "mrr_at_10": 0.824683, + "mrr_at_20": 0.826078, + "mrr_at_100": 0.826647, + "mrr_at_1000": 0.826692, + "nauc_ndcg_at_1_max": 0.263358, + "nauc_ndcg_at_1_std": -0.452328, + "nauc_ndcg_at_1_diff1": 0.854019, + "nauc_ndcg_at_3_max": 0.308478, + "nauc_ndcg_at_3_std": -0.510416, + "nauc_ndcg_at_3_diff1": 0.827476, + "nauc_ndcg_at_5_max": 0.321261, + "nauc_ndcg_at_5_std": -0.508989, + "nauc_ndcg_at_5_diff1": 0.82737, + "nauc_ndcg_at_10_max": 0.313502, + "nauc_ndcg_at_10_std": -0.513735, + "nauc_ndcg_at_10_diff1": 0.83118, + "nauc_ndcg_at_20_max": 0.307117, + "nauc_ndcg_at_20_std": -0.520463, + "nauc_ndcg_at_20_diff1": 0.835066, + "nauc_ndcg_at_100_max": 0.305807, + "nauc_ndcg_at_100_std": -0.501728, + "nauc_ndcg_at_100_diff1": 0.835096, + "nauc_ndcg_at_1000_max": 0.304424, + "nauc_ndcg_at_1000_std": -0.500081, + "nauc_ndcg_at_1000_diff1": 0.835972, + "nauc_map_at_1_max": 0.244835, + "nauc_map_at_1_std": -0.430079, + "nauc_map_at_1_diff1": 0.849647, + "nauc_map_at_3_max": 0.293906, + "nauc_map_at_3_std": -0.493435, + "nauc_map_at_3_diff1": 0.835758, + "nauc_map_at_5_max": 0.302545, + "nauc_map_at_5_std": -0.49555, + "nauc_map_at_5_diff1": 0.83639, + "nauc_map_at_10_max": 0.299479, + "nauc_map_at_10_std": -0.496309, + "nauc_map_at_10_diff1": 0.837839, + "nauc_map_at_20_max": 0.298042, + "nauc_map_at_20_std": -0.497498, + "nauc_map_at_20_diff1": 0.838768, + "nauc_map_at_100_max": 0.298079, + "nauc_map_at_100_std": -0.495087, + "nauc_map_at_100_diff1": 0.838797, + "nauc_map_at_1000_max": 0.298027, + "nauc_map_at_1000_std": -0.494975, + "nauc_map_at_1000_diff1": 0.838835, + "nauc_recall_at_1_max": 0.244835, + "nauc_recall_at_1_std": -0.430079, + "nauc_recall_at_1_diff1": 0.849647, + "nauc_recall_at_3_max": 0.353582, + "nauc_recall_at_3_std": -0.572486, + "nauc_recall_at_3_diff1": 0.79054, + "nauc_recall_at_5_max": 0.416805, + "nauc_recall_at_5_std": -0.586062, + "nauc_recall_at_5_diff1": 0.771442, + "nauc_recall_at_10_max": 0.420158, + "nauc_recall_at_10_std": -0.686798, + "nauc_recall_at_10_diff1": 0.773805, + "nauc_recall_at_20_max": 0.388501, + "nauc_recall_at_20_std": -0.933863, + "nauc_recall_at_20_diff1": 0.802753, + "nauc_recall_at_100_max": 0.471904, + "nauc_recall_at_100_std": -0.658171, + "nauc_recall_at_100_diff1": 0.746282, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_1000_diff1": NaN, + "nauc_precision_at_1_max": 0.263358, + "nauc_precision_at_1_std": -0.452328, + "nauc_precision_at_1_diff1": 0.854019, + "nauc_precision_at_3_max": 0.286143, + "nauc_precision_at_3_std": -0.394624, + "nauc_precision_at_3_diff1": 0.40499, + "nauc_precision_at_5_max": 0.279022, + "nauc_precision_at_5_std": -0.293595, + "nauc_precision_at_5_diff1": 0.213449, + "nauc_precision_at_10_max": 0.218099, + "nauc_precision_at_10_std": -0.194092, + "nauc_precision_at_10_diff1": 0.02392, + "nauc_precision_at_20_max": 0.156641, + "nauc_precision_at_20_std": -0.130772, + "nauc_precision_at_20_diff1": -0.117775, + "nauc_precision_at_100_max": 0.076731, + "nauc_precision_at_100_std": 0.120321, + "nauc_precision_at_100_diff1": -0.315668, + "nauc_precision_at_1000_max": 0.038973, + "nauc_precision_at_1000_std": 0.181339, + "nauc_precision_at_1000_diff1": -0.378983, + "nauc_mrr_at_1_max": 0.263358, + "nauc_mrr_at_1_std": -0.452328, + "nauc_mrr_at_1_diff1": 0.854019, + "nauc_mrr_at_3_max": 0.299282, + "nauc_mrr_at_3_std": -0.494441, + "nauc_mrr_at_3_diff1": 0.834893, + "nauc_mrr_at_5_max": 0.300837, + "nauc_mrr_at_5_std": -0.48756, + "nauc_mrr_at_5_diff1": 0.834529, + "nauc_mrr_at_10_max": 0.296749, + "nauc_mrr_at_10_std": -0.490076, + "nauc_mrr_at_10_diff1": 0.836656, + "nauc_mrr_at_20_max": 0.294756, + "nauc_mrr_at_20_std": -0.492094, + "nauc_mrr_at_20_diff1": 0.837557, + "nauc_mrr_at_100_max": 0.294766, + "nauc_mrr_at_100_std": -0.489944, + "nauc_mrr_at_100_diff1": 0.83761, + "nauc_mrr_at_1000_max": 0.294734, + "nauc_mrr_at_1000_std": -0.489948, + "nauc_mrr_at_1000_diff1": 0.837653, + "hit_rate_at_1": 0.75414, + "hit_rate_at_3": 0.88227, + "hit_rate_at_5": 0.92009, + "hit_rate_at_10": 0.95319, + "hit_rate_at_20": 0.97352, + "hit_rate_at_100": 0.99338, + "hit_rate_at_1000": 1.0, + "main_score": 0.85282, + "hf_subset": "pt", + "languages": [ + "por-Latn" + ] + } + ] + }, + "evaluation_time": 2596.5890583992004, + "kg_co2_emissions": null, + "date": 1777987299.657786 +} \ No newline at end of file diff --git a/results/nvidia__llama-nemotron-embed-1b-v2/b4caa8456edd360b3b4e938d94ed4398dd437fad/experiments/trust_remote_code_True/model_meta.json b/results/nvidia__llama-nemotron-embed-1b-v2/b4caa8456edd360b3b4e938d94ed4398dd437fad/experiments/trust_remote_code_True/model_meta.json new file mode 100644 index 0000000000..b82bdba343 --- /dev/null +++ b/results/nvidia__llama-nemotron-embed-1b-v2/b4caa8456edd360b3b4e938d94ed4398dd437fad/experiments/trust_remote_code_True/model_meta.json @@ -0,0 +1,47 @@ +{ + "loader_kwargs": { + "trust_remote_code": true, + "device": "cuda" + }, + "name": "nvidia/llama-nemotron-embed-1b-v2", + "revision": "b4caa8456edd360b3b4e938d94ed4398dd437fad", + "release_date": "2025-10-16", + "languages": null, + "n_parameters": 1235814400, + "n_active_parameters_override": null, + "n_embedding_parameters": 262668288, + "memory_usage_mb": 2357, + "max_tokens": 131072, + "embed_dim": 2048, + "license": null, + "open_weights": null, + "public_training_code": null, + "public_training_data": null, + "framework": [ + "Sentence Transformers", + "PyTorch", + "Transformers", + "safetensors" + ], + "reference": "https://huggingface.co/nvidia/llama-nemotron-embed-1b-v2", + "similarity_fn_name": "cosine", + "use_instructions": null, + "training_datasets": null, + "adapted_from": null, + "superseded_by": null, + "modalities": [ + "text" + ], + "model_type": [ + "dense" + ], + "citation": null, + "contacts": null, + "experiment_kwargs": { + "trust_remote_code": true + }, + "output_dtypes": null, + "extra_requirements_groups": null, + "loader": "SentenceTransformerEncoderWrapper", + "is_cross_encoder": false +} \ No newline at end of file diff --git a/run_models.py b/run_models.py new file mode 100644 index 0000000000..66399ded50 --- /dev/null +++ b/run_models.py @@ -0,0 +1,72 @@ +# /// script +# requires-python = "==3.12.*" +# dependencies = [ +# "mteb[peft]>=2.12.37", +# "transformers>=4.57.0", +# "peft>=0.15.2", +# "torch", +# "flash-attn @ https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.7.16/flash_attn-2.8.3+cu128torch2.7-cp312-cp312-linux_x86_64.whl", +# ] +# +# [tool.uv.sources] +# mteb = { git = "https://github.com/embeddings-benchmark/mteb", branch="fix-private-sets" } +# torch = { index = "pytorch-cu128" } +# +# [[tool.uv.index]] +# name = "pytorch-cu128" +# url = "https://download.pytorch.org/whl/cu128" +# explicit = true +# /// +""" +requires login: +uv pip install huggingface_hub +hf auth login + +install from git should not be nec., but the latest version does not seem to be in pypi (will check later) +also missed had to fix the private sets (so using branch fix-private-sets) +""" +import logging +from pathlib import Path + +import mteb + +# setup logger: +# logging.basicConfig( +# format="%(asctime)s - %(levelname)s - %(message)s", +# level=logging.INFO, +# ) + +task = mteb.get_tasks( + [ + "LexRetrieval.v1", + "GlobalNewsRetrieval", + "PublicNewsRetrieval", + "ElasticKBRetrieval", + ] +) + +model_names = [ + "jinaai/jina-embeddings-v5-text-small", # missing elastic +] + +cache_path = Path(__file__).parent +cache = mteb.ResultCache(cache_path) +assert cache.get_models() + +model_metas = mteb.get_model_metas(model_names) + +# for model_meta in model_metas: +# model_meta._check_requirements() # peft + +# not implemented in mteb yet, so directly get model and evaluate here - sentence trf. compatible though. +# model_name= "nvidia/llama-nemotron-embed-1b-v2" +# model = mteb.get_model("nvidia/llama-nemotron-embed-1b-v2", trust_remote_code=True, device="cuda") +# print(model_name) +# res = mteb.evaluate(model, task, cache=cache, raise_error=False, encode_kwargs={"batch_size": 8}) + +for model in model_metas: + print(model.name) + model.revision = "dd76d535f5447ca3897a9c893fb1e612ead98192" # overwrite revision + # res = mteb.evaluate(model, task, cache=cache, encode_kwargs={"batch_size": 8}, raise_error=True) + res = mteb.evaluate(model, task, cache=cache, encode_kwargs={"batch_size": 8}) +