Skip to content

evaluate multiple models on the new private sets #511

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
KennethEnevoldsen wants to merge 15 commits into from
Closed

evaluate multiple models on the new private sets #511

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
2eee156
evaluate multiple models on the new private sets
May 3, 2026
14a1c1a
all models seem to run with the specified batch size, submitting firs…
May 3, 2026
bb6212a
add res
May 4, 2026
bebe1a3
add res
May 4, 2026
bfc75ef
add res
May 4, 2026
0ce3ecb
add res
May 4, 2026
8e16ffa
add res
May 5, 2026
4477bf0
add more res
May 5, 2026
51e395b
add res
May 5, 2026
05c6616
add res
May 5, 2026
353a65f
add jina results
May 5, 2026
1d7319f
add more res
May 5, 2026
3f0cb77
add res
May 5, 2026
0e22192
add res
May 5, 2026
ef66e11
add last res
May 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
324 changes: 324 additions & 0 deletions results/BAAI__bge-m3/5617a9f61b028005a4858fdac845db406aefb181/ElasticKBRetrieval.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,324 @@
{
"dataset_revision": "21bdf1e024bf7c9f46720017559ce2f8c6116507",
"task_name": "ElasticKBRetrieval",
"mteb_version": "2.12.38",
"scores": {
"synthetic_test": [
{
"ndcg_at_1": 0.70213,
"ndcg_at_3": 0.56287,
"ndcg_at_5": 0.53684,
"ndcg_at_10": 0.51354,
"ndcg_at_20": 0.51233,
"ndcg_at_100": 0.55395,
"ndcg_at_1000": 0.61713,
"map_at_1": 0.22897,
"map_at_3": 0.28765,
"map_at_5": 0.3055,
"map_at_10": 0.32613,
"map_at_20": 0.34619,
"map_at_100": 0.37307,
"map_at_1000": 0.38407,
"recall_at_1": 0.22897,
"recall_at_3": 0.3188,
"recall_at_5": 0.35703,
"recall_at_10": 0.40616,
"recall_at_20": 0.47311,
"recall_at_100": 0.64121,
"recall_at_1000": 0.84787,
"accuracy": 0.22897,
"precision_at_1": 0.70213,
"precision_at_3": 0.41489,
"precision_at_5": 0.32979,
"precision_at_10": 0.23245,
"precision_at_20": 0.16755,
"precision_at_100": 0.06553,
"precision_at_1000": 0.01234,
"mrr_at_1": 0.702128,
"mrr_at_3": 0.782801,
"mrr_at_5": 0.793706,
"mrr_at_10": 0.797716,
"mrr_at_20": 0.799809,
"mrr_at_100": 0.800236,
"mrr_at_1000": 0.800236,
"nauc_ndcg_at_1_max": 0.03637,
"nauc_ndcg_at_1_std": -0.155315,
"nauc_ndcg_at_1_diff1": 0.393593,
"nauc_ndcg_at_3_max": 0.101207,
"nauc_ndcg_at_3_std": 0.059933,
"nauc_ndcg_at_3_diff1": 0.133793,
"nauc_ndcg_at_5_max": 0.10129,
"nauc_ndcg_at_5_std": 0.063464,
"nauc_ndcg_at_5_diff1": 0.172103,
"nauc_ndcg_at_10_max": 0.13233,
"nauc_ndcg_at_10_std": 0.08067,
"nauc_ndcg_at_10_diff1": 0.262056,
"nauc_ndcg_at_20_max": 0.177488,
"nauc_ndcg_at_20_std": 0.101258,
"nauc_ndcg_at_20_diff1": 0.292539,
"nauc_ndcg_at_100_max": 0.238721,
"nauc_ndcg_at_100_std": 0.174771,
"nauc_ndcg_at_100_diff1": 0.304403,
"nauc_ndcg_at_1000_max": 0.223684,
"nauc_ndcg_at_1000_std": 0.186361,
"nauc_ndcg_at_1000_diff1": 0.266568,
"nauc_map_at_1_max": 0.09017,
"nauc_map_at_1_std": -0.011468,
"nauc_map_at_1_diff1": 0.49284,
"nauc_map_at_3_max": 0.122122,
"nauc_map_at_3_std": -0.012008,
"nauc_map_at_3_diff1": 0.428711,
"nauc_map_at_5_max": 0.119363,
"nauc_map_at_5_std": -0.013921,
"nauc_map_at_5_diff1": 0.41813,
"nauc_map_at_10_max": 0.139708,
"nauc_map_at_10_std": 0.006488,
"nauc_map_at_10_diff1": 0.406346,
"nauc_map_at_20_max": 0.178808,
"nauc_map_at_20_std": 0.052469,
"nauc_map_at_20_diff1": 0.374139,
"nauc_map_at_100_max": 0.20081,
"nauc_map_at_100_std": 0.100564,
"nauc_map_at_100_diff1": 0.348661,
"nauc_map_at_1000_max": 0.197797,
"nauc_map_at_1000_std": 0.109939,
"nauc_map_at_1000_diff1": 0.340116,
"nauc_recall_at_1_max": 0.09017,
"nauc_recall_at_1_std": -0.011468,
"nauc_recall_at_1_diff1": 0.49284,
"nauc_recall_at_3_max": 0.106298,
"nauc_recall_at_3_std": -0.002766,
"nauc_recall_at_3_diff1": 0.366062,
"nauc_recall_at_5_max": 0.097266,
"nauc_recall_at_5_std": -0.015098,
"nauc_recall_at_5_diff1": 0.357176,
"nauc_recall_at_10_max": 0.13274,
"nauc_recall_at_10_std": 0.018272,
"nauc_recall_at_10_diff1": 0.350881,
"nauc_recall_at_20_max": 0.198021,
"nauc_recall_at_20_std": 0.098643,
"nauc_recall_at_20_diff1": 0.266127,
"nauc_recall_at_100_max": 0.237467,
"nauc_recall_at_100_std": 0.235715,
"nauc_recall_at_100_diff1": 0.219191,
"nauc_recall_at_1000_max": 0.194755,
"nauc_recall_at_1000_std": 0.259497,
"nauc_recall_at_1000_diff1": 0.214908,
"nauc_precision_at_1_max": 0.03637,
"nauc_precision_at_1_std": -0.155315,
"nauc_precision_at_1_diff1": 0.393593,
"nauc_precision_at_3_max": 0.102194,
"nauc_precision_at_3_std": 0.115327,
"nauc_precision_at_3_diff1": -0.232675,
"nauc_precision_at_5_max": 0.051921,
"nauc_precision_at_5_std": 0.125716,
"nauc_precision_at_5_diff1": -0.311005,
"nauc_precision_at_10_max": 0.078458,
"nauc_precision_at_10_std": 0.209922,
"nauc_precision_at_10_diff1": -0.328814,
"nauc_precision_at_20_max": 0.134614,
"nauc_precision_at_20_std": 0.300578,
"nauc_precision_at_20_diff1": -0.390179,
"nauc_precision_at_100_max": -0.012795,
"nauc_precision_at_100_std": 0.249381,
"nauc_precision_at_100_diff1": -0.371273,
"nauc_precision_at_1000_max": -0.173382,
"nauc_precision_at_1000_std": 0.063768,
"nauc_precision_at_1000_diff1": -0.303265,
"nauc_mrr_at_1_max": 0.03637,
"nauc_mrr_at_1_std": -0.155315,
"nauc_mrr_at_1_diff1": 0.393593,
"nauc_mrr_at_3_max": 0.087938,
"nauc_mrr_at_3_std": -0.079861,
"nauc_mrr_at_3_diff1": 0.326316,
"nauc_mrr_at_5_max": 0.080416,
"nauc_mrr_at_5_std": -0.089234,
"nauc_mrr_at_5_diff1": 0.347435,
"nauc_mrr_at_10_max": 0.084397,
"nauc_mrr_at_10_std": -0.089832,
"nauc_mrr_at_10_diff1": 0.359597,
"nauc_mrr_at_20_max": 0.079692,
"nauc_mrr_at_20_std": -0.094003,
"nauc_mrr_at_20_diff1": 0.354783,
"nauc_mrr_at_100_max": 0.078293,
"nauc_mrr_at_100_std": -0.094719,
"nauc_mrr_at_100_diff1": 0.354029,
"nauc_mrr_at_1000_max": 0.078293,
"nauc_mrr_at_1000_std": -0.094719,
"nauc_mrr_at_1000_diff1": 0.354029,
"hit_rate_at_1": 0.70213,
"hit_rate_at_3": 0.87766,
"hit_rate_at_5": 0.92553,
"hit_rate_at_10": 0.95213,
"hit_rate_at_20": 0.98404,
"hit_rate_at_100": 1.0,
"hit_rate_at_1000": 1.0,
"main_score": 0.51354,
"hf_subset": "en",
"languages": [
"eng-Latn"
]
}
],
"real_chat_test": [
{
"ndcg_at_1": 0.73276,
"ndcg_at_3": 0.65531,
"ndcg_at_5": 0.61008,
"ndcg_at_10": 0.55386,
"ndcg_at_20": 0.51278,
"ndcg_at_100": 0.46226,
"ndcg_at_1000": 0.54149,
"map_at_1": 0.05417,
"map_at_3": 0.09937,
"map_at_5": 0.11647,
"map_at_10": 0.13877,
"map_at_20": 0.15862,
"map_at_100": 0.2014,
"map_at_1000": 0.25157,
"recall_at_1": 0.05417,
"recall_at_3": 0.11056,
"recall_at_5": 0.13696,
"recall_at_10": 0.17956,
"recall_at_20": 0.22769,
"recall_at_100": 0.37549,
"recall_at_1000": 0.69322,
"accuracy": 0.05417,
"precision_at_1": 0.73276,
"precision_at_3": 0.62069,
"precision_at_5": 0.54828,
"precision_at_10": 0.45216,
"precision_at_20": 0.36789,
"precision_at_100": 0.21707,
"precision_at_1000": 0.07642,
"mrr_at_1": 0.732759,
"mrr_at_3": 0.801724,
"mrr_at_5": 0.809483,
"mrr_at_10": 0.814426,
"mrr_at_20": 0.816045,
"mrr_at_100": 0.816574,
"mrr_at_1000": 0.816599,
"nauc_ndcg_at_1_max": 0.392414,
"nauc_ndcg_at_1_std": 0.239732,
"nauc_ndcg_at_1_diff1": 0.408702,
"nauc_ndcg_at_3_max": 0.31895,
"nauc_ndcg_at_3_std": 0.168499,
"nauc_ndcg_at_3_diff1": 0.194345,
"nauc_ndcg_at_5_max": 0.311744,
"nauc_ndcg_at_5_std": 0.183467,
"nauc_ndcg_at_5_diff1": 0.140404,
"nauc_ndcg_at_10_max": 0.338781,
"nauc_ndcg_at_10_std": 0.251598,
"nauc_ndcg_at_10_diff1": 0.137225,
"nauc_ndcg_at_20_max": 0.350291,
"nauc_ndcg_at_20_std": 0.276095,
"nauc_ndcg_at_20_diff1": 0.148665,
"nauc_ndcg_at_100_max": 0.433396,
"nauc_ndcg_at_100_std": 0.28703,
"nauc_ndcg_at_100_diff1": 0.243558,
"nauc_ndcg_at_1000_max": 0.423155,
"nauc_ndcg_at_1000_std": 0.328531,
"nauc_ndcg_at_1000_diff1": 0.255891,
"nauc_map_at_1_max": 0.372885,
"nauc_map_at_1_std": -0.034218,
"nauc_map_at_1_diff1": 0.564564,
"nauc_map_at_3_max": 0.323696,
"nauc_map_at_3_std": -0.087526,
"nauc_map_at_3_diff1": 0.428836,
"nauc_map_at_5_max": 0.323596,
"nauc_map_at_5_std": -0.095468,
"nauc_map_at_5_diff1": 0.414677,
"nauc_map_at_10_max": 0.351467,
"nauc_map_at_10_std": -0.051137,
"nauc_map_at_10_diff1": 0.405714,
"nauc_map_at_20_max": 0.358121,
"nauc_map_at_20_std": -0.024856,
"nauc_map_at_20_diff1": 0.390702,
"nauc_map_at_100_max": 0.405742,
"nauc_map_at_100_std": 0.093437,
"nauc_map_at_100_diff1": 0.350073,
"nauc_map_at_1000_max": 0.4367,
"nauc_map_at_1000_std": 0.226577,
"nauc_map_at_1000_diff1": 0.299833,
"nauc_recall_at_1_max": 0.372885,
"nauc_recall_at_1_std": -0.034218,
"nauc_recall_at_1_diff1": 0.564564,
"nauc_recall_at_3_max": 0.26701,
"nauc_recall_at_3_std": -0.114567,
"nauc_recall_at_3_diff1": 0.380972,
"nauc_recall_at_5_max": 0.251532,
"nauc_recall_at_5_std": -0.126484,
"nauc_recall_at_5_diff1": 0.356845,
"nauc_recall_at_10_max": 0.297609,
"nauc_recall_at_10_std": -0.049554,
"nauc_recall_at_10_diff1": 0.349745,
"nauc_recall_at_20_max": 0.289278,
"nauc_recall_at_20_std": -0.024984,
"nauc_recall_at_20_diff1": 0.316856,
"nauc_recall_at_100_max": 0.310333,
"nauc_recall_at_100_std": 0.110186,
"nauc_recall_at_100_diff1": 0.245096,
"nauc_recall_at_1000_max": 0.286375,
"nauc_recall_at_1000_std": 0.283108,
"nauc_recall_at_1000_diff1": 0.152341,
"nauc_precision_at_1_max": 0.392414,
"nauc_precision_at_1_std": 0.239732,
"nauc_precision_at_1_diff1": 0.408702,
"nauc_precision_at_3_max": 0.263014,
"nauc_precision_at_3_std": 0.165473,
"nauc_precision_at_3_diff1": 0.082503,
"nauc_precision_at_5_max": 0.228145,
"nauc_precision_at_5_std": 0.192415,
"nauc_precision_at_5_diff1": -0.016206,
"nauc_precision_at_10_max": 0.19358,
"nauc_precision_at_10_std": 0.281694,
"nauc_precision_at_10_diff1": -0.094217,
"nauc_precision_at_20_max": 0.11416,
"nauc_precision_at_20_std": 0.329075,
"nauc_precision_at_20_diff1": -0.169462,
"nauc_precision_at_100_max": 0.079117,
"nauc_precision_at_100_std": 0.372067,
"nauc_precision_at_100_diff1": -0.195606,
"nauc_precision_at_1000_max": 0.005204,
"nauc_precision_at_1000_std": 0.233734,
"nauc_precision_at_1000_diff1": -0.169497,
"nauc_mrr_at_1_max": 0.392414,
"nauc_mrr_at_1_std": 0.239732,
"nauc_mrr_at_1_diff1": 0.408702,
"nauc_mrr_at_3_max": 0.41073,
"nauc_mrr_at_3_std": 0.232272,
"nauc_mrr_at_3_diff1": 0.409098,
"nauc_mrr_at_5_max": 0.397771,
"nauc_mrr_at_5_std": 0.225726,
"nauc_mrr_at_5_diff1": 0.4051,
"nauc_mrr_at_10_max": 0.407635,
"nauc_mrr_at_10_std": 0.224537,
"nauc_mrr_at_10_diff1": 0.410387,
"nauc_mrr_at_20_max": 0.407774,
"nauc_mrr_at_20_std": 0.224568,
"nauc_mrr_at_20_diff1": 0.410494,
"nauc_mrr_at_100_max": 0.407187,
"nauc_mrr_at_100_std": 0.226996,
"nauc_mrr_at_100_diff1": 0.409546,
"nauc_mrr_at_1000_max": 0.407195,
"nauc_mrr_at_1000_std": 0.226923,
"nauc_mrr_at_1000_diff1": 0.409689,
"hit_rate_at_1": 0.73276,
"hit_rate_at_3": 0.88793,
"hit_rate_at_5": 0.92241,
"hit_rate_at_10": 0.9569,
"hit_rate_at_20": 0.97845,
"hit_rate_at_100": 0.99569,
"hit_rate_at_1000": 1.0,
"main_score": 0.55386,
"hf_subset": "en",
"languages": [
"eng-Latn"
]
}
]
},
"evaluation_time": 470.54601192474365,
"kg_co2_emissions": null,
"date": 1777843190.525151
}
Loading
Loading