From 79d4b542657c4e6647cf3e9c949fc2d41b351665 Mon Sep 17 00:00:00 2001 From: Isaac Chung Date: Sun, 14 Jul 2024 08:14:34 +0000 Subject: [PATCH] add english retrieval results --- results/bm25s/0_1_10/ArguAna.json | 158 ++++++ .../0_1_10/CQADupstackAndroidRetrieval.json | 158 ++++++ .../0_1_10/CQADupstackEnglishRetrieval.json | 158 ++++++ .../0_1_10/CQADupstackGamingRetrieval.json | 158 ++++++ .../bm25s/0_1_10/CQADupstackGisRetrieval.json | 158 ++++++ .../CQADupstackMathematicaRetrieval.json | 158 ++++++ .../0_1_10/CQADupstackPhysicsRetrieval.json | 158 ++++++ .../CQADupstackProgrammersRetrieval.json | 158 ++++++ .../0_1_10/CQADupstackStatsRetrieval.json | 158 ++++++ .../bm25s/0_1_10/CQADupstackTexRetrieval.json | 158 ++++++ .../0_1_10/CQADupstackUnixRetrieval.json | 158 ++++++ .../CQADupstackWebmastersRetrieval.json | 158 ++++++ .../0_1_10/CQADupstackWordpressRetrieval.json | 158 ++++++ results/bm25s/0_1_10/ClimateFEVER.json | 158 ++++++ results/bm25s/0_1_10/DBPedia.json | 307 ++++++++++++ results/bm25s/0_1_10/FEVER.json | 456 ++++++++++++++++++ results/bm25s/0_1_10/FiQA2018.json | 456 ++++++++++++++++++ results/bm25s/0_1_10/HotpotQA.json | 456 ++++++++++++++++++ results/bm25s/0_1_10/MSMARCO.json | 456 ++++++++++++++++++ results/bm25s/0_1_10/NFCorpus.json | 158 ++++++ results/bm25s/0_1_10/NQ.json | 158 ++++++ results/bm25s/0_1_10/QuoraRetrieval.json | 307 ++++++++++++ results/bm25s/0_1_10/SCIDOCS.json | 158 ++++++ results/bm25s/0_1_10/SciFact.json | 307 ++++++++++++ results/bm25s/0_1_10/TRECCOVID.json | 158 ++++++ results/bm25s/0_1_10/Touche2020.json | 158 ++++++ results/bm25s/0_1_10/model_meta.json | 1 + 27 files changed, 5748 insertions(+) create mode 100644 results/bm25s/0_1_10/ArguAna.json create mode 100644 results/bm25s/0_1_10/CQADupstackAndroidRetrieval.json create mode 100644 results/bm25s/0_1_10/CQADupstackEnglishRetrieval.json create mode 100644 results/bm25s/0_1_10/CQADupstackGamingRetrieval.json create mode 100644 results/bm25s/0_1_10/CQADupstackGisRetrieval.json create mode 100644 results/bm25s/0_1_10/CQADupstackMathematicaRetrieval.json create mode 100644 results/bm25s/0_1_10/CQADupstackPhysicsRetrieval.json create mode 100644 results/bm25s/0_1_10/CQADupstackProgrammersRetrieval.json create mode 100644 results/bm25s/0_1_10/CQADupstackStatsRetrieval.json create mode 100644 results/bm25s/0_1_10/CQADupstackTexRetrieval.json create mode 100644 results/bm25s/0_1_10/CQADupstackUnixRetrieval.json create mode 100644 results/bm25s/0_1_10/CQADupstackWebmastersRetrieval.json create mode 100644 results/bm25s/0_1_10/CQADupstackWordpressRetrieval.json create mode 100644 results/bm25s/0_1_10/ClimateFEVER.json create mode 100644 results/bm25s/0_1_10/DBPedia.json create mode 100644 results/bm25s/0_1_10/FEVER.json create mode 100644 results/bm25s/0_1_10/FiQA2018.json create mode 100644 results/bm25s/0_1_10/HotpotQA.json create mode 100644 results/bm25s/0_1_10/MSMARCO.json create mode 100644 results/bm25s/0_1_10/NFCorpus.json create mode 100644 results/bm25s/0_1_10/NQ.json create mode 100644 results/bm25s/0_1_10/QuoraRetrieval.json create mode 100644 results/bm25s/0_1_10/SCIDOCS.json create mode 100644 results/bm25s/0_1_10/SciFact.json create mode 100644 results/bm25s/0_1_10/TRECCOVID.json create mode 100644 results/bm25s/0_1_10/Touche2020.json create mode 100644 results/bm25s/0_1_10/model_meta.json diff --git a/results/bm25s/0_1_10/ArguAna.json b/results/bm25s/0_1_10/ArguAna.json new file mode 100644 index 0000000000..5e2220458f --- /dev/null +++ b/results/bm25s/0_1_10/ArguAna.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "c22ab2a51041ffd869aaddef7af8d8215647e41a", + "evaluation_time": 5.660568475723267, + "kg_co2_emissions": null, + "mteb_version": "1.12.22", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.49276, + "map_at_1": 0.24609, + "map_at_10": 0.40338, + "map_at_100": 0.41342, + "map_at_1000": 0.41355, + "map_at_20": 0.41109, + "map_at_3": 0.351, + "map_at_5": 0.38129, + "mrr_at_1": 0.25604551920341395, + "mrr_at_10": 0.40720748944433144, + "mrr_at_100": 0.4170380307552231, + "mrr_at_1000": 0.41717558754366546, + "mrr_at_20": 0.4146724769107446, + "mrr_at_3": 0.3560929350403027, + "mrr_at_5": 0.3846491228070172, + "nauc_map_at_1000_diff1": 0.021886370060854626, + "nauc_map_at_1000_max": -0.050589239073000766, + "nauc_map_at_1000_std": -0.047255238785213136, + "nauc_map_at_100_diff1": 0.021991848534549694, + "nauc_map_at_100_max": -0.05036037681015677, + "nauc_map_at_100_std": -0.046991639959645434, + "nauc_map_at_10_diff1": 0.01944695569349535, + "nauc_map_at_10_max": -0.05135259543708522, + "nauc_map_at_10_std": -0.04932007591740704, + "nauc_map_at_1_diff1": 0.03701924027832177, + "nauc_map_at_1_max": -0.10073437912329636, + "nauc_map_at_1_std": -0.09714228089183849, + "nauc_map_at_20_diff1": 0.021074707557638035, + "nauc_map_at_20_max": -0.04862337290352727, + "nauc_map_at_20_std": -0.04543320111520716, + "nauc_map_at_3_diff1": 0.018847825117445496, + "nauc_map_at_3_max": -0.0633251115176233, + "nauc_map_at_3_std": -0.06117393852247131, + "nauc_map_at_5_diff1": 0.020660933587840602, + "nauc_map_at_5_max": -0.05322561221432506, + "nauc_map_at_5_std": -0.05039034499825575, + "nauc_mrr_at_1000_diff1": -0.0069916774432518996, + "nauc_mrr_at_1000_max": -0.05333586260967066, + "nauc_mrr_at_1000_std": -0.04418644510256895, + "nauc_mrr_at_100_diff1": -0.006874087285323051, + "nauc_mrr_at_100_max": -0.05310772579098728, + "nauc_mrr_at_100_std": -0.0439264910005182, + "nauc_mrr_at_10_diff1": -0.008377656435482347, + "nauc_mrr_at_10_max": -0.05392075810974482, + "nauc_mrr_at_10_std": -0.04651614968325181, + "nauc_mrr_at_1_diff1": 0.005327314532673115, + "nauc_mrr_at_1_max": -0.08512604423534934, + "nauc_mrr_at_1_std": -0.07834121981070495, + "nauc_mrr_at_20_diff1": -0.0077329431928073195, + "nauc_mrr_at_20_max": -0.05152246898546906, + "nauc_mrr_at_20_std": -0.04257929430502551, + "nauc_mrr_at_3_diff1": -0.004856572445127299, + "nauc_mrr_at_3_max": -0.06240231043434506, + "nauc_mrr_at_3_std": -0.055716396330877466, + "nauc_mrr_at_5_diff1": -0.006755969959898004, + "nauc_mrr_at_5_max": -0.056478414555520336, + "nauc_mrr_at_5_std": -0.04801654234480827, + "nauc_ndcg_at_1000_diff1": 0.022385904672540317, + "nauc_ndcg_at_1000_max": -0.037668490148391166, + "nauc_ndcg_at_1000_std": -0.032296367627030594, + "nauc_ndcg_at_100_diff1": 0.026383707483814727, + "nauc_ndcg_at_100_max": -0.02961668256385589, + "nauc_ndcg_at_100_std": -0.02343964973523178, + "nauc_ndcg_at_10_diff1": 0.013312910506790721, + "nauc_ndcg_at_10_max": -0.029936910520231143, + "nauc_ndcg_at_10_std": -0.030239455934052435, + "nauc_ndcg_at_1_diff1": 0.03701924027832177, + "nauc_ndcg_at_1_max": -0.10073437912329636, + "nauc_ndcg_at_1_std": -0.09714228089183849, + "nauc_ndcg_at_20_diff1": 0.020332954448972276, + "nauc_ndcg_at_20_max": -0.01659652124941782, + "nauc_ndcg_at_20_std": -0.012171448030433511, + "nauc_ndcg_at_3_diff1": 0.014181473782307842, + "nauc_ndcg_at_3_max": -0.055017566324620126, + "nauc_ndcg_at_3_std": -0.05379142189559777, + "nauc_ndcg_at_5_diff1": 0.016594438418392792, + "nauc_ndcg_at_5_max": -0.03635193601664064, + "nauc_ndcg_at_5_std": -0.03396739766745262, + "nauc_precision_at_1000_diff1": 0.04548813525954615, + "nauc_precision_at_1000_max": -0.03741762492642651, + "nauc_precision_at_1000_std": 0.3434893045428403, + "nauc_precision_at_100_diff1": 0.26131642925368015, + "nauc_precision_at_100_max": 0.427745624881254, + "nauc_precision_at_100_std": 0.5507113012498237, + "nauc_precision_at_10_diff1": -0.015344585938079884, + "nauc_precision_at_10_max": 0.0717330694399166, + "nauc_precision_at_10_std": 0.05857802419902309, + "nauc_precision_at_1_diff1": 0.03701924027832177, + "nauc_precision_at_1_max": -0.10073437912329636, + "nauc_precision_at_1_std": -0.09714228089183849, + "nauc_precision_at_20_diff1": 0.030689672237653447, + "nauc_precision_at_20_max": 0.27286736721211624, + "nauc_precision_at_20_std": 0.29357201667169436, + "nauc_precision_at_3_diff1": 0.0015611753676343357, + "nauc_precision_at_3_max": -0.03300014690374674, + "nauc_precision_at_3_std": -0.034468293667097305, + "nauc_precision_at_5_diff1": 0.004303597872377514, + "nauc_precision_at_5_max": 0.01723887778063657, + "nauc_precision_at_5_std": 0.01822794811792894, + "nauc_recall_at_1000_diff1": 0.045488135259533935, + "nauc_recall_at_1000_max": -0.0374176249263954, + "nauc_recall_at_1000_std": 0.3434893045428401, + "nauc_recall_at_100_diff1": 0.26131642925368626, + "nauc_recall_at_100_max": 0.42774562488126144, + "nauc_recall_at_100_std": 0.550711301249821, + "nauc_recall_at_10_diff1": -0.015344585938079393, + "nauc_recall_at_10_max": 0.07173306943991657, + "nauc_recall_at_10_std": 0.05857802419902259, + "nauc_recall_at_1_diff1": 0.03701924027832177, + "nauc_recall_at_1_max": -0.10073437912329636, + "nauc_recall_at_1_std": -0.09714228089183849, + "nauc_recall_at_20_diff1": 0.030689672237654887, + "nauc_recall_at_20_max": 0.27286736721211635, + "nauc_recall_at_20_std": 0.29357201667169563, + "nauc_recall_at_3_diff1": 0.0015611753676343357, + "nauc_recall_at_3_max": -0.033000146903746364, + "nauc_recall_at_3_std": -0.03446829366709721, + "nauc_recall_at_5_diff1": 0.004303597872377939, + "nauc_recall_at_5_max": 0.017238877780636937, + "nauc_recall_at_5_std": 0.018227948117928935, + "ndcg_at_1": 0.24609, + "ndcg_at_10": 0.49276, + "ndcg_at_100": 0.53527, + "ndcg_at_1000": 0.53883, + "ndcg_at_20": 0.51975, + "ndcg_at_3": 0.38507, + "ndcg_at_5": 0.43946, + "precision_at_1": 0.24609, + "precision_at_10": 0.07795, + "precision_at_100": 0.00965, + "precision_at_1000": 0.00099, + "precision_at_20": 0.0442, + "precision_at_3": 0.16121, + "precision_at_5": 0.12304, + "recall_at_1": 0.24609, + "recall_at_10": 0.77952, + "recall_at_100": 0.96515, + "recall_at_1000": 0.99289, + "recall_at_20": 0.88407, + "recall_at_3": 0.48364, + "recall_at_5": 0.61522 + } + ] + }, + "task_name": "ArguAna" +} \ No newline at end of file diff --git a/results/bm25s/0_1_10/CQADupstackAndroidRetrieval.json b/results/bm25s/0_1_10/CQADupstackAndroidRetrieval.json new file mode 100644 index 0000000000..40258c3dda --- /dev/null +++ b/results/bm25s/0_1_10/CQADupstackAndroidRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "f46a197baaae43b4f621051089b82a364682dfeb", + "evaluation_time": 4.090010404586792, + "kg_co2_emissions": null, + "mteb_version": "1.12.22", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.39693, + "map_at_1": 0.25945, + "map_at_10": 0.34614, + "map_at_100": 0.35768, + "map_at_1000": 0.35892, + "map_at_20": 0.35218, + "map_at_3": 0.31913, + "map_at_5": 0.33394, + "mrr_at_1": 0.3218884120171674, + "mrr_at_10": 0.39754524604310026, + "mrr_at_100": 0.4049561715182821, + "mrr_at_1000": 0.40555047405524475, + "mrr_at_20": 0.4013601001878152, + "mrr_at_3": 0.3745827372436815, + "mrr_at_5": 0.3881020505484025, + "nauc_map_at_1000_diff1": 0.5401042788373607, + "nauc_map_at_1000_max": 0.46446941292616584, + "nauc_map_at_1000_std": 0.1448534325440925, + "nauc_map_at_100_diff1": 0.5399915323525181, + "nauc_map_at_100_max": 0.4641570929554848, + "nauc_map_at_100_std": 0.1444170242192052, + "nauc_map_at_10_diff1": 0.544123002626852, + "nauc_map_at_10_max": 0.46633938701722494, + "nauc_map_at_10_std": 0.14139150808244658, + "nauc_map_at_1_diff1": 0.576256760811547, + "nauc_map_at_1_max": 0.4539253680333111, + "nauc_map_at_1_std": 0.13886111230903941, + "nauc_map_at_20_diff1": 0.5406412193228595, + "nauc_map_at_20_max": 0.4640427163014816, + "nauc_map_at_20_std": 0.14192478286159718, + "nauc_map_at_3_diff1": 0.5565388392220246, + "nauc_map_at_3_max": 0.46657852693639296, + "nauc_map_at_3_std": 0.1414241365632173, + "nauc_map_at_5_diff1": 0.5493623117253394, + "nauc_map_at_5_max": 0.4661875641780336, + "nauc_map_at_5_std": 0.1398507592644627, + "nauc_mrr_at_1000_diff1": 0.5444244071501962, + "nauc_mrr_at_1000_max": 0.4693117952655278, + "nauc_mrr_at_1000_std": 0.15224424700571026, + "nauc_mrr_at_100_diff1": 0.5442781921808686, + "nauc_mrr_at_100_max": 0.4689664786770939, + "nauc_mrr_at_100_std": 0.15180516785703468, + "nauc_mrr_at_10_diff1": 0.546510598770559, + "nauc_mrr_at_10_max": 0.47178794029012106, + "nauc_mrr_at_10_std": 0.1514325118602757, + "nauc_mrr_at_1_diff1": 0.587458620032547, + "nauc_mrr_at_1_max": 0.4839179169052132, + "nauc_mrr_at_1_std": 0.15314613502076443, + "nauc_mrr_at_20_diff1": 0.5438642084003957, + "nauc_mrr_at_20_max": 0.4685892515983097, + "nauc_mrr_at_20_std": 0.15000004328956787, + "nauc_mrr_at_3_diff1": 0.5611595454121813, + "nauc_mrr_at_3_max": 0.4819413050014325, + "nauc_mrr_at_3_std": 0.1622140602258072, + "nauc_mrr_at_5_diff1": 0.5503681989476072, + "nauc_mrr_at_5_max": 0.47437329907861125, + "nauc_mrr_at_5_std": 0.1545662953584148, + "nauc_ndcg_at_1000_diff1": 0.5166912488184727, + "nauc_ndcg_at_1000_max": 0.46091945324253136, + "nauc_ndcg_at_1000_std": 0.1646539581699839, + "nauc_ndcg_at_100_diff1": 0.510840931307867, + "nauc_ndcg_at_100_max": 0.44987548346310813, + "nauc_ndcg_at_100_std": 0.1524670393894962, + "nauc_ndcg_at_10_diff1": 0.521250964842959, + "nauc_ndcg_at_10_max": 0.4589169947078968, + "nauc_ndcg_at_10_std": 0.13809521772037603, + "nauc_ndcg_at_1_diff1": 0.587458620032547, + "nauc_ndcg_at_1_max": 0.4839179169052132, + "nauc_ndcg_at_1_std": 0.15314613502076443, + "nauc_ndcg_at_20_diff1": 0.513078444102445, + "nauc_ndcg_at_20_max": 0.44854573945143655, + "nauc_ndcg_at_20_std": 0.1383467153046283, + "nauc_ndcg_at_3_diff1": 0.5489683241106224, + "nauc_ndcg_at_3_max": 0.47346936137111983, + "nauc_ndcg_at_3_std": 0.15026629693286153, + "nauc_ndcg_at_5_diff1": 0.532738240644177, + "nauc_ndcg_at_5_max": 0.46327370065068596, + "nauc_ndcg_at_5_std": 0.13948065084537742, + "nauc_precision_at_1000_diff1": -0.016452908247665064, + "nauc_precision_at_1000_max": -0.012001214641875885, + "nauc_precision_at_1000_std": -0.007429596498602477, + "nauc_precision_at_100_diff1": 0.043750177743398136, + "nauc_precision_at_100_max": 0.11840841801001449, + "nauc_precision_at_100_std": 0.09423170869894978, + "nauc_precision_at_10_diff1": 0.25775160299406186, + "nauc_precision_at_10_max": 0.31370734595865013, + "nauc_precision_at_10_std": 0.11623518438348741, + "nauc_precision_at_1_diff1": 0.587458620032547, + "nauc_precision_at_1_max": 0.4839179169052132, + "nauc_precision_at_1_std": 0.15314613502076443, + "nauc_precision_at_20_diff1": 0.18275415322892, + "nauc_precision_at_20_max": 0.24214692223337636, + "nauc_precision_at_20_std": 0.1026496751611631, + "nauc_precision_at_3_diff1": 0.4372477308856027, + "nauc_precision_at_3_max": 0.4206846005073202, + "nauc_precision_at_3_std": 0.14288990628895132, + "nauc_precision_at_5_diff1": 0.35601343726042023, + "nauc_precision_at_5_max": 0.3695758503618295, + "nauc_precision_at_5_std": 0.12143486994349435, + "nauc_recall_at_1000_diff1": 0.3290972545754859, + "nauc_recall_at_1000_max": 0.47809942001313116, + "nauc_recall_at_1000_std": 0.4554009253629691, + "nauc_recall_at_100_diff1": 0.3441698585821832, + "nauc_recall_at_100_max": 0.34250128812089764, + "nauc_recall_at_100_std": 0.17695741822462932, + "nauc_recall_at_10_diff1": 0.4326595941825119, + "nauc_recall_at_10_max": 0.4000349281281342, + "nauc_recall_at_10_std": 0.10268244219902738, + "nauc_recall_at_1_diff1": 0.576256760811547, + "nauc_recall_at_1_max": 0.4539253680333111, + "nauc_recall_at_1_std": 0.13886111230903941, + "nauc_recall_at_20_diff1": 0.39204221492763575, + "nauc_recall_at_20_max": 0.3566711669315542, + "nauc_recall_at_20_std": 0.10167314401440865, + "nauc_recall_at_3_diff1": 0.5100005611427266, + "nauc_recall_at_3_max": 0.44693489202542486, + "nauc_recall_at_3_std": 0.14034968317991875, + "nauc_recall_at_5_diff1": 0.4730348111968922, + "nauc_recall_at_5_max": 0.42656953405586634, + "nauc_recall_at_5_std": 0.12175169946828732, + "ndcg_at_1": 0.32189, + "ndcg_at_10": 0.39693, + "ndcg_at_100": 0.44568, + "ndcg_at_1000": 0.47072, + "ndcg_at_20": 0.41324, + "ndcg_at_3": 0.35782, + "ndcg_at_5": 0.37523, + "precision_at_1": 0.32189, + "precision_at_10": 0.07382, + "precision_at_100": 0.01209, + "precision_at_1000": 0.00173, + "precision_at_20": 0.04313, + "precision_at_3": 0.17024, + "precision_at_5": 0.1216, + "recall_at_1": 0.25945, + "recall_at_10": 0.49321, + "recall_at_100": 0.70705, + "recall_at_1000": 0.87218, + "recall_at_20": 0.55432, + "recall_at_3": 0.37485, + "recall_at_5": 0.42614 + } + ] + }, + "task_name": "CQADupstackAndroidRetrieval" +} \ No newline at end of file diff --git a/results/bm25s/0_1_10/CQADupstackEnglishRetrieval.json b/results/bm25s/0_1_10/CQADupstackEnglishRetrieval.json new file mode 100644 index 0000000000..7a6da7fa44 --- /dev/null +++ b/results/bm25s/0_1_10/CQADupstackEnglishRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ad9991cb51e31e31e430383c75ffb2885547b5f0", + "evaluation_time": 8.094563961029053, + "kg_co2_emissions": null, + "mteb_version": "1.12.22", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.34746, + "map_at_1": 0.24087, + "map_at_10": 0.30727, + "map_at_100": 0.31646, + "map_at_1000": 0.31744, + "map_at_20": 0.31197, + "map_at_3": 0.28743, + "map_at_5": 0.29854, + "mrr_at_1": 0.2961783439490446, + "mrr_at_10": 0.3550017692852087, + "mrr_at_100": 0.36159055890607705, + "mrr_at_1000": 0.3621414895752817, + "mrr_at_20": 0.35863541739877725, + "mrr_at_3": 0.33895966029723984, + "mrr_at_5": 0.34787685774946914, + "nauc_map_at_1000_diff1": 0.507605383077198, + "nauc_map_at_1000_max": 0.3899260526214966, + "nauc_map_at_1000_std": 0.03383009648282197, + "nauc_map_at_100_diff1": 0.5077301343252304, + "nauc_map_at_100_max": 0.389984792170568, + "nauc_map_at_100_std": 0.03358691067403598, + "nauc_map_at_10_diff1": 0.512498419355122, + "nauc_map_at_10_max": 0.39133569645135224, + "nauc_map_at_10_std": 0.029210159838665, + "nauc_map_at_1_diff1": 0.568248594624497, + "nauc_map_at_1_max": 0.3795846927024012, + "nauc_map_at_1_std": 0.004867971752939656, + "nauc_map_at_20_diff1": 0.5095023906734152, + "nauc_map_at_20_max": 0.3901861107337471, + "nauc_map_at_20_std": 0.030780434479465945, + "nauc_map_at_3_diff1": 0.5265517612313007, + "nauc_map_at_3_max": 0.3936142218717427, + "nauc_map_at_3_std": 0.020717705748870226, + "nauc_map_at_5_diff1": 0.5191490205767499, + "nauc_map_at_5_max": 0.394275221093101, + "nauc_map_at_5_std": 0.027710686784447703, + "nauc_mrr_at_1000_diff1": 0.5094223241781185, + "nauc_mrr_at_1000_max": 0.3962789373115541, + "nauc_mrr_at_1000_std": 0.05698378476226783, + "nauc_mrr_at_100_diff1": 0.5093561357655734, + "nauc_mrr_at_100_max": 0.39627018066290437, + "nauc_mrr_at_100_std": 0.05695955731108714, + "nauc_mrr_at_10_diff1": 0.5122170627542646, + "nauc_mrr_at_10_max": 0.3983811457887141, + "nauc_mrr_at_10_std": 0.05602514736517242, + "nauc_mrr_at_1_diff1": 0.5646285014051087, + "nauc_mrr_at_1_max": 0.41390472973850695, + "nauc_mrr_at_1_std": 0.04421870331640276, + "nauc_mrr_at_20_diff1": 0.5099795808649789, + "nauc_mrr_at_20_max": 0.39681327813901285, + "nauc_mrr_at_20_std": 0.0565103526119274, + "nauc_mrr_at_3_diff1": 0.520970804975272, + "nauc_mrr_at_3_max": 0.40446399826491913, + "nauc_mrr_at_3_std": 0.05462304501946109, + "nauc_mrr_at_5_diff1": 0.5189004458324016, + "nauc_mrr_at_5_max": 0.40398717612025653, + "nauc_mrr_at_5_std": 0.058791744606224765, + "nauc_ndcg_at_1000_diff1": 0.4697290438868409, + "nauc_ndcg_at_1000_max": 0.37596679149302525, + "nauc_ndcg_at_1000_std": 0.05624887253105248, + "nauc_ndcg_at_100_diff1": 0.471508428090201, + "nauc_ndcg_at_100_max": 0.37739475544249323, + "nauc_ndcg_at_100_std": 0.05500829470873566, + "nauc_ndcg_at_10_diff1": 0.4865080103469214, + "nauc_ndcg_at_10_max": 0.3834703941906101, + "nauc_ndcg_at_10_std": 0.04187595089109166, + "nauc_ndcg_at_1_diff1": 0.5646285014051087, + "nauc_ndcg_at_1_max": 0.41390472973850695, + "nauc_ndcg_at_1_std": 0.04421870331640276, + "nauc_ndcg_at_20_diff1": 0.47844039505848757, + "nauc_ndcg_at_20_max": 0.378616149494826, + "nauc_ndcg_at_20_std": 0.044748780321361536, + "nauc_ndcg_at_3_diff1": 0.5094795315639791, + "nauc_ndcg_at_3_max": 0.4021882559055949, + "nauc_ndcg_at_3_std": 0.04171989896187942, + "nauc_ndcg_at_5_diff1": 0.5006057505702911, + "nauc_ndcg_at_5_max": 0.3956724765549609, + "nauc_ndcg_at_5_std": 0.045169241647689165, + "nauc_precision_at_1000_diff1": -0.07602500742619589, + "nauc_precision_at_1000_max": 0.041085461021834545, + "nauc_precision_at_1000_std": 0.1301151611882119, + "nauc_precision_at_100_diff1": 0.030853633318456662, + "nauc_precision_at_100_max": 0.13206658455462825, + "nauc_precision_at_100_std": 0.13828208061415528, + "nauc_precision_at_10_diff1": 0.23131918368522322, + "nauc_precision_at_10_max": 0.27974249244848204, + "nauc_precision_at_10_std": 0.10154825412595228, + "nauc_precision_at_1_diff1": 0.5646285014051087, + "nauc_precision_at_1_max": 0.41390472973850695, + "nauc_precision_at_1_std": 0.04421870331640276, + "nauc_precision_at_20_diff1": 0.16001146462363977, + "nauc_precision_at_20_max": 0.22864736309163908, + "nauc_precision_at_20_std": 0.11142023865464977, + "nauc_precision_at_3_diff1": 0.37691891634441405, + "nauc_precision_at_3_max": 0.37855328599642063, + "nauc_precision_at_3_std": 0.08424082714110596, + "nauc_precision_at_5_diff1": 0.3188349858356941, + "nauc_precision_at_5_max": 0.34998996694995294, + "nauc_precision_at_5_std": 0.10361898016997154, + "nauc_recall_at_1000_diff1": 0.24676450589415974, + "nauc_recall_at_1000_max": 0.26399480321640706, + "nauc_recall_at_1000_std": 0.11956730162773192, + "nauc_recall_at_100_diff1": 0.3154217971111324, + "nauc_recall_at_100_max": 0.3032241916775269, + "nauc_recall_at_100_std": 0.1064048665721694, + "nauc_recall_at_10_diff1": 0.40482397079919585, + "nauc_recall_at_10_max": 0.3371645819528801, + "nauc_recall_at_10_std": 0.04564401920803145, + "nauc_recall_at_1_diff1": 0.568248594624497, + "nauc_recall_at_1_max": 0.3795846927024012, + "nauc_recall_at_1_std": 0.004867971752939656, + "nauc_recall_at_20_diff1": 0.3705485047094261, + "nauc_recall_at_20_max": 0.3175907540700319, + "nauc_recall_at_20_std": 0.05647470735091848, + "nauc_recall_at_3_diff1": 0.4748515291996907, + "nauc_recall_at_3_max": 0.37714589244822583, + "nauc_recall_at_3_std": 0.0320397565875811, + "nauc_recall_at_5_diff1": 0.447994660888753, + "nauc_recall_at_5_max": 0.36873002829813856, + "nauc_recall_at_5_std": 0.05251198599410889, + "ndcg_at_1": 0.29618, + "ndcg_at_10": 0.34746, + "ndcg_at_100": 0.3861, + "ndcg_at_1000": 0.40884, + "ndcg_at_20": 0.36059, + "ndcg_at_3": 0.31821, + "ndcg_at_5": 0.33136, + "precision_at_1": 0.29618, + "precision_at_10": 0.06153, + "precision_at_100": 0.01, + "precision_at_1000": 0.0014, + "precision_at_20": 0.03618, + "precision_at_3": 0.14926, + "precision_at_5": 0.10318, + "recall_at_1": 0.24087, + "recall_at_10": 0.41667, + "recall_at_100": 0.58505, + "recall_at_1000": 0.74283, + "recall_at_20": 0.46293, + "recall_at_3": 0.32821, + "recall_at_5": 0.36733 + } + ] + }, + "task_name": "CQADupstackEnglishRetrieval" +} \ No newline at end of file diff --git a/results/bm25s/0_1_10/CQADupstackGamingRetrieval.json b/results/bm25s/0_1_10/CQADupstackGamingRetrieval.json new file mode 100644 index 0000000000..2dea43bd55 --- /dev/null +++ b/results/bm25s/0_1_10/CQADupstackGamingRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "4885aa143210c98657558c04aaf3dc47cfb54340", + "evaluation_time": 8.47208833694458, + "kg_co2_emissions": null, + "mteb_version": "1.12.22", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.48471, + "map_at_1": 0.34211, + "map_at_10": 0.43796, + "map_at_100": 0.44757, + "map_at_1000": 0.44826, + "map_at_20": 0.44317, + "map_at_3": 0.41254, + "map_at_5": 0.427, + "mrr_at_1": 0.3887147335423197, + "mrr_at_10": 0.46845573966263643, + "mrr_at_100": 0.4757504087295201, + "mrr_at_1000": 0.47621327006089603, + "mrr_at_20": 0.4726400745920921, + "mrr_at_3": 0.4470219435736677, + "mrr_at_5": 0.4596865203761758, + "nauc_map_at_1000_diff1": 0.5761307400486146, + "nauc_map_at_1000_max": 0.46816385006201483, + "nauc_map_at_1000_std": 0.12916150313583052, + "nauc_map_at_100_diff1": 0.5759769949149924, + "nauc_map_at_100_max": 0.468172512392129, + "nauc_map_at_100_std": 0.12900121940919534, + "nauc_map_at_10_diff1": 0.5769758877085143, + "nauc_map_at_10_max": 0.46602751446883667, + "nauc_map_at_10_std": 0.1227923396746526, + "nauc_map_at_1_diff1": 0.6065016225673595, + "nauc_map_at_1_max": 0.4140368428696759, + "nauc_map_at_1_std": 0.07549717146966647, + "nauc_map_at_20_diff1": 0.5764407602641605, + "nauc_map_at_20_max": 0.4676681441037194, + "nauc_map_at_20_std": 0.12659317713579937, + "nauc_map_at_3_diff1": 0.5826152034457897, + "nauc_map_at_3_max": 0.45839225316109067, + "nauc_map_at_3_std": 0.10770324029028253, + "nauc_map_at_5_diff1": 0.5777168905069933, + "nauc_map_at_5_max": 0.46337596470361314, + "nauc_map_at_5_std": 0.11660891610390298, + "nauc_mrr_at_1000_diff1": 0.580617775469161, + "nauc_mrr_at_1000_max": 0.47266783693453746, + "nauc_mrr_at_1000_std": 0.13684224539062814, + "nauc_mrr_at_100_diff1": 0.580529037049751, + "nauc_mrr_at_100_max": 0.47271995709278547, + "nauc_mrr_at_100_std": 0.13682563534337597, + "nauc_mrr_at_10_diff1": 0.5800665850409541, + "nauc_mrr_at_10_max": 0.47114436097554524, + "nauc_mrr_at_10_std": 0.13312367159836444, + "nauc_mrr_at_1_diff1": 0.6152349016677855, + "nauc_mrr_at_1_max": 0.4435399820692492, + "nauc_mrr_at_1_std": 0.10130110404749716, + "nauc_mrr_at_20_diff1": 0.5802146959581068, + "nauc_mrr_at_20_max": 0.47246589748878387, + "nauc_mrr_at_20_std": 0.13588088130768852, + "nauc_mrr_at_3_diff1": 0.5860252932558163, + "nauc_mrr_at_3_max": 0.4688314904905922, + "nauc_mrr_at_3_std": 0.12627651847700247, + "nauc_mrr_at_5_diff1": 0.5801503941973827, + "nauc_mrr_at_5_max": 0.47159579545537134, + "nauc_mrr_at_5_std": 0.13205038869053787, + "nauc_ndcg_at_1000_diff1": 0.5643842117517708, + "nauc_ndcg_at_1000_max": 0.4844429135548687, + "nauc_ndcg_at_1000_std": 0.16249901169433076, + "nauc_ndcg_at_100_diff1": 0.5609957098264071, + "nauc_ndcg_at_100_max": 0.48542022380390243, + "nauc_ndcg_at_100_std": 0.1614421514621831, + "nauc_ndcg_at_10_diff1": 0.5627092786985185, + "nauc_ndcg_at_10_max": 0.47753538835655895, + "nauc_ndcg_at_10_std": 0.13947383730517507, + "nauc_ndcg_at_1_diff1": 0.6152349016677855, + "nauc_ndcg_at_1_max": 0.4435399820692492, + "nauc_ndcg_at_1_std": 0.10130110404749716, + "nauc_ndcg_at_20_diff1": 0.562153235959237, + "nauc_ndcg_at_20_max": 0.4831888962729388, + "nauc_ndcg_at_20_std": 0.1511761387875299, + "nauc_ndcg_at_3_diff1": 0.5708280710453237, + "nauc_ndcg_at_3_max": 0.46882727900084836, + "nauc_ndcg_at_3_std": 0.12410002900379433, + "nauc_ndcg_at_5_diff1": 0.5625288678236077, + "nauc_ndcg_at_5_max": 0.47437559133479923, + "nauc_ndcg_at_5_std": 0.13209614387276047, + "nauc_precision_at_1000_diff1": -0.027020250515043236, + "nauc_precision_at_1000_max": 0.0958763775129772, + "nauc_precision_at_1000_std": 0.20687745706477345, + "nauc_precision_at_100_diff1": 0.0749849089413402, + "nauc_precision_at_100_max": 0.21932755443374988, + "nauc_precision_at_100_std": 0.24055583484767704, + "nauc_precision_at_10_diff1": 0.2888762000858519, + "nauc_precision_at_10_max": 0.3739896940946824, + "nauc_precision_at_10_std": 0.2129258532790748, + "nauc_precision_at_1_diff1": 0.6152349016677855, + "nauc_precision_at_1_max": 0.4435399820692492, + "nauc_precision_at_1_std": 0.10130110404749716, + "nauc_precision_at_20_diff1": 0.21405717340829422, + "nauc_precision_at_20_max": 0.32835652512704494, + "nauc_precision_at_20_std": 0.23127026379175938, + "nauc_precision_at_3_diff1": 0.43046430925252843, + "nauc_precision_at_3_max": 0.4403137892909935, + "nauc_precision_at_3_std": 0.1680762823048696, + "nauc_precision_at_5_diff1": 0.36052216268243337, + "nauc_precision_at_5_max": 0.41429161263038744, + "nauc_precision_at_5_std": 0.1897339938942342, + "nauc_recall_at_1000_diff1": 0.4333419922274795, + "nauc_recall_at_1000_max": 0.5810065161927389, + "nauc_recall_at_1000_std": 0.4677030181333733, + "nauc_recall_at_100_diff1": 0.45106618373545826, + "nauc_recall_at_100_max": 0.5384866963282271, + "nauc_recall_at_100_std": 0.31292069636105113, + "nauc_recall_at_10_diff1": 0.4995706309493555, + "nauc_recall_at_10_max": 0.4898079409545446, + "nauc_recall_at_10_std": 0.17133720068720573, + "nauc_recall_at_1_diff1": 0.6065016225673595, + "nauc_recall_at_1_max": 0.4140368428696759, + "nauc_recall_at_1_std": 0.07549717146966647, + "nauc_recall_at_20_diff1": 0.487616481994268, + "nauc_recall_at_20_max": 0.5121444991597566, + "nauc_recall_at_20_std": 0.2198266503132677, + "nauc_recall_at_3_diff1": 0.5347917580681395, + "nauc_recall_at_3_max": 0.4694958000671067, + "nauc_recall_at_3_std": 0.12259750756240557, + "nauc_recall_at_5_diff1": 0.5040164667807334, + "nauc_recall_at_5_max": 0.480957075779029, + "nauc_recall_at_5_std": 0.1510407844662707, + "ndcg_at_1": 0.38871, + "ndcg_at_10": 0.48471, + "ndcg_at_100": 0.52763, + "ndcg_at_1000": 0.54359, + "ndcg_at_20": 0.50071, + "ndcg_at_3": 0.44093, + "ndcg_at_5": 0.46252, + "precision_at_1": 0.38871, + "precision_at_10": 0.07567, + "precision_at_100": 0.0106, + "precision_at_1000": 0.00125, + "precision_at_20": 0.04254, + "precision_at_3": 0.1931, + "precision_at_5": 0.13154, + "recall_at_1": 0.34211, + "recall_at_10": 0.5938, + "recall_at_100": 0.78485, + "recall_at_1000": 0.89932, + "recall_at_20": 0.65261, + "recall_at_3": 0.47508, + "recall_at_5": 0.52852 + } + ] + }, + "task_name": "CQADupstackGamingRetrieval" +} \ No newline at end of file diff --git a/results/bm25s/0_1_10/CQADupstackGisRetrieval.json b/results/bm25s/0_1_10/CQADupstackGisRetrieval.json new file mode 100644 index 0000000000..91bc157b05 --- /dev/null +++ b/results/bm25s/0_1_10/CQADupstackGisRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "5003b3064772da1887988e05400cf3806fe491f2", + "evaluation_time": 7.761858701705933, + "kg_co2_emissions": null, + "mteb_version": "1.12.22", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.3083, + "map_at_1": 0.2141, + "map_at_10": 0.27295, + "map_at_100": 0.28207, + "map_at_1000": 0.28312, + "map_at_20": 0.27748, + "map_at_3": 0.25419, + "map_at_5": 0.26578, + "mrr_at_1": 0.2327683615819209, + "mrr_at_10": 0.2913021253699219, + "mrr_at_100": 0.2998882260094255, + "mrr_at_1000": 0.3007797849409865, + "mrr_at_20": 0.2957631708797946, + "mrr_at_3": 0.27269303201506606, + "mrr_at_5": 0.2838229755178908, + "nauc_map_at_1000_diff1": 0.4980640570364222, + "nauc_map_at_1000_max": 0.44292717336065973, + "nauc_map_at_1000_std": 0.17339079774683427, + "nauc_map_at_100_diff1": 0.4975248468217703, + "nauc_map_at_100_max": 0.442604239311236, + "nauc_map_at_100_std": 0.17306019870989856, + "nauc_map_at_10_diff1": 0.5026129991221717, + "nauc_map_at_10_max": 0.4450176598446959, + "nauc_map_at_10_std": 0.17110103591925108, + "nauc_map_at_1_diff1": 0.5509207486770765, + "nauc_map_at_1_max": 0.4337763148022456, + "nauc_map_at_1_std": 0.13829655960917028, + "nauc_map_at_20_diff1": 0.4989891949479384, + "nauc_map_at_20_max": 0.4421708272157593, + "nauc_map_at_20_std": 0.1702041849771554, + "nauc_map_at_3_diff1": 0.5168773385891859, + "nauc_map_at_3_max": 0.4448844935546559, + "nauc_map_at_3_std": 0.16239628145953175, + "nauc_map_at_5_diff1": 0.5062841159919966, + "nauc_map_at_5_max": 0.4481382278665953, + "nauc_map_at_5_std": 0.17016634286652868, + "nauc_mrr_at_1000_diff1": 0.48744927825395323, + "nauc_mrr_at_1000_max": 0.44996140811558744, + "nauc_mrr_at_1000_std": 0.19491629399583105, + "nauc_mrr_at_100_diff1": 0.4868792744012703, + "nauc_mrr_at_100_max": 0.44986864722936337, + "nauc_mrr_at_100_std": 0.19488765299918265, + "nauc_mrr_at_10_diff1": 0.4910434154842184, + "nauc_mrr_at_10_max": 0.4521271290749364, + "nauc_mrr_at_10_std": 0.19388159994529136, + "nauc_mrr_at_1_diff1": 0.5430736496652693, + "nauc_mrr_at_1_max": 0.4469578771804769, + "nauc_mrr_at_1_std": 0.16343519021720068, + "nauc_mrr_at_20_diff1": 0.48778232723922943, + "nauc_mrr_at_20_max": 0.449264196162218, + "nauc_mrr_at_20_std": 0.1926177028369078, + "nauc_mrr_at_3_diff1": 0.5063222748646822, + "nauc_mrr_at_3_max": 0.4550934724897892, + "nauc_mrr_at_3_std": 0.1879499939710128, + "nauc_mrr_at_5_diff1": 0.49627559363517243, + "nauc_mrr_at_5_max": 0.4571322920854383, + "nauc_mrr_at_5_std": 0.19490190640879793, + "nauc_ndcg_at_1000_diff1": 0.4641280261278409, + "nauc_ndcg_at_1000_max": 0.43749197033497966, + "nauc_ndcg_at_1000_std": 0.1992236643595059, + "nauc_ndcg_at_100_diff1": 0.45361235044015513, + "nauc_ndcg_at_100_max": 0.4331992610888067, + "nauc_ndcg_at_100_std": 0.19671082184800662, + "nauc_ndcg_at_10_diff1": 0.47517259978061904, + "nauc_ndcg_at_10_max": 0.44302845200682583, + "nauc_ndcg_at_10_std": 0.18535738531091434, + "nauc_ndcg_at_1_diff1": 0.5430736496652693, + "nauc_ndcg_at_1_max": 0.4469578771804769, + "nauc_ndcg_at_1_std": 0.16343519021720068, + "nauc_ndcg_at_20_diff1": 0.4628693730450653, + "nauc_ndcg_at_20_max": 0.4321964207076464, + "nauc_ndcg_at_20_std": 0.18113142197294538, + "nauc_ndcg_at_3_diff1": 0.5017668251422565, + "nauc_ndcg_at_3_max": 0.44966537541168283, + "nauc_ndcg_at_3_std": 0.1753276319399132, + "nauc_ndcg_at_5_diff1": 0.48424108455443104, + "nauc_ndcg_at_5_max": 0.4522989650087252, + "nauc_ndcg_at_5_std": 0.18607126122134696, + "nauc_precision_at_1000_diff1": 0.05173082086132333, + "nauc_precision_at_1000_max": 0.2243852827771864, + "nauc_precision_at_1000_std": 0.247693381747973, + "nauc_precision_at_100_diff1": 0.18010918097435172, + "nauc_precision_at_100_max": 0.33712312488890356, + "nauc_precision_at_100_std": 0.27991956100634774, + "nauc_precision_at_10_diff1": 0.35985268559167066, + "nauc_precision_at_10_max": 0.4349310862300089, + "nauc_precision_at_10_std": 0.2395934592228728, + "nauc_precision_at_1_diff1": 0.5430736496652693, + "nauc_precision_at_1_max": 0.4469578771804769, + "nauc_precision_at_1_std": 0.16343519021720068, + "nauc_precision_at_20_diff1": 0.3030191106392338, + "nauc_precision_at_20_max": 0.39112287178133803, + "nauc_precision_at_20_std": 0.22343740509574495, + "nauc_precision_at_3_diff1": 0.44760916085907804, + "nauc_precision_at_3_max": 0.45829171611390773, + "nauc_precision_at_3_std": 0.21186555862630582, + "nauc_precision_at_5_diff1": 0.38979904069562565, + "nauc_precision_at_5_max": 0.4581352691101432, + "nauc_precision_at_5_std": 0.23724149036868894, + "nauc_recall_at_1000_diff1": 0.29238300998044037, + "nauc_recall_at_1000_max": 0.3444659791996781, + "nauc_recall_at_1000_std": 0.30883746767771325, + "nauc_recall_at_100_diff1": 0.2782526952243364, + "nauc_recall_at_100_max": 0.3556796517275251, + "nauc_recall_at_100_std": 0.2530441738866371, + "nauc_recall_at_10_diff1": 0.39965756508832734, + "nauc_recall_at_10_max": 0.417273412880478, + "nauc_recall_at_10_std": 0.19698506183782893, + "nauc_recall_at_1_diff1": 0.5509207486770765, + "nauc_recall_at_1_max": 0.4337763148022456, + "nauc_recall_at_1_std": 0.13829655960917028, + "nauc_recall_at_20_diff1": 0.3518900224704063, + "nauc_recall_at_20_max": 0.3732920146392407, + "nauc_recall_at_20_std": 0.179394059733766, + "nauc_recall_at_3_diff1": 0.46901840154312824, + "nauc_recall_at_3_max": 0.4439030421371201, + "nauc_recall_at_3_std": 0.1799546926166945, + "nauc_recall_at_5_diff1": 0.4270793622344529, + "nauc_recall_at_5_max": 0.4458512233165119, + "nauc_recall_at_5_std": 0.20186111643445662, + "ndcg_at_1": 0.23277, + "ndcg_at_10": 0.3083, + "ndcg_at_100": 0.35679, + "ndcg_at_1000": 0.38315, + "ndcg_at_20": 0.32388, + "ndcg_at_3": 0.27171, + "ndcg_at_5": 0.29134, + "precision_at_1": 0.23277, + "precision_at_10": 0.04565, + "precision_at_100": 0.00739, + "precision_at_1000": 0.001, + "precision_at_20": 0.02621, + "precision_at_3": 0.11262, + "precision_at_5": 0.07932, + "recall_at_1": 0.2141, + "recall_at_10": 0.40021, + "recall_at_100": 0.63133, + "recall_at_1000": 0.82781, + "recall_at_20": 0.45905, + "recall_at_3": 0.30165, + "recall_at_5": 0.34939 + } + ] + }, + "task_name": "CQADupstackGisRetrieval" +} \ No newline at end of file diff --git a/results/bm25s/0_1_10/CQADupstackMathematicaRetrieval.json b/results/bm25s/0_1_10/CQADupstackMathematicaRetrieval.json new file mode 100644 index 0000000000..3da315f839 --- /dev/null +++ b/results/bm25s/0_1_10/CQADupstackMathematicaRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "90fceea13679c63fe563ded68f3b6f06e50061de", + "evaluation_time": 4.615939617156982, + "kg_co2_emissions": null, + "mteb_version": "1.12.22", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.22017, + "map_at_1": 0.12558, + "map_at_10": 0.18037, + "map_at_100": 0.19031, + "map_at_1000": 0.19149, + "map_at_20": 0.1855, + "map_at_3": 0.16133, + "map_at_5": 0.1719, + "mrr_at_1": 0.15796019900497513, + "mrr_at_10": 0.21826482666034897, + "mrr_at_100": 0.22732177276591092, + "mrr_at_1000": 0.22817953478030142, + "mrr_at_20": 0.22301255236684506, + "mrr_at_3": 0.19879767827529024, + "mrr_at_5": 0.20912106135986733, + "nauc_map_at_1000_diff1": 0.4295317438704705, + "nauc_map_at_1000_max": 0.4338294909965671, + "nauc_map_at_1000_std": 0.18860339369295726, + "nauc_map_at_100_diff1": 0.4293874708363056, + "nauc_map_at_100_max": 0.4338159909262415, + "nauc_map_at_100_std": 0.18831853924682462, + "nauc_map_at_10_diff1": 0.43426728788805863, + "nauc_map_at_10_max": 0.431493816420856, + "nauc_map_at_10_std": 0.1785582753295133, + "nauc_map_at_1_diff1": 0.49479342920699904, + "nauc_map_at_1_max": 0.4328222440912369, + "nauc_map_at_1_std": 0.14008045475803277, + "nauc_map_at_20_diff1": 0.4299419738359594, + "nauc_map_at_20_max": 0.4326758145133673, + "nauc_map_at_20_std": 0.18440600310411048, + "nauc_map_at_3_diff1": 0.45608123909633314, + "nauc_map_at_3_max": 0.43995785220453, + "nauc_map_at_3_std": 0.1671452844782646, + "nauc_map_at_5_diff1": 0.43943930960012695, + "nauc_map_at_5_max": 0.4326739526807951, + "nauc_map_at_5_std": 0.171449803140176, + "nauc_mrr_at_1000_diff1": 0.4170581868922272, + "nauc_mrr_at_1000_max": 0.407159732899823, + "nauc_mrr_at_1000_std": 0.16966357730028583, + "nauc_mrr_at_100_diff1": 0.41679247103681333, + "nauc_mrr_at_100_max": 0.4070909523537896, + "nauc_mrr_at_100_std": 0.16951389330842442, + "nauc_mrr_at_10_diff1": 0.42105412075075954, + "nauc_mrr_at_10_max": 0.40709654669285705, + "nauc_mrr_at_10_std": 0.16672682113073028, + "nauc_mrr_at_1_diff1": 0.49148043282667364, + "nauc_mrr_at_1_max": 0.4162281278183343, + "nauc_mrr_at_1_std": 0.13353245801794972, + "nauc_mrr_at_20_diff1": 0.4174888975854485, + "nauc_mrr_at_20_max": 0.4069341562800931, + "nauc_mrr_at_20_std": 0.16812501284716885, + "nauc_mrr_at_3_diff1": 0.4398644774534084, + "nauc_mrr_at_3_max": 0.4177920806393565, + "nauc_mrr_at_3_std": 0.1602041515877134, + "nauc_mrr_at_5_diff1": 0.4256890384767003, + "nauc_mrr_at_5_max": 0.40789537033424594, + "nauc_mrr_at_5_std": 0.1595878434339859, + "nauc_ndcg_at_1000_diff1": 0.3861885382183784, + "nauc_ndcg_at_1000_max": 0.4273237045045555, + "nauc_ndcg_at_1000_std": 0.22610512185179968, + "nauc_ndcg_at_100_diff1": 0.3854069234468305, + "nauc_ndcg_at_100_max": 0.4291537938836079, + "nauc_ndcg_at_100_std": 0.22545045285330428, + "nauc_ndcg_at_10_diff1": 0.4042689014733785, + "nauc_ndcg_at_10_max": 0.4215214669437669, + "nauc_ndcg_at_10_std": 0.19341832968483041, + "nauc_ndcg_at_1_diff1": 0.49148043282667364, + "nauc_ndcg_at_1_max": 0.4162281278183343, + "nauc_ndcg_at_1_std": 0.13353245801794972, + "nauc_ndcg_at_20_diff1": 0.39167506815389097, + "nauc_ndcg_at_20_max": 0.4238476436079149, + "nauc_ndcg_at_20_std": 0.20725844415390438, + "nauc_ndcg_at_3_diff1": 0.4462448552192997, + "nauc_ndcg_at_3_max": 0.4409457656566489, + "nauc_ndcg_at_3_std": 0.17665718707352512, + "nauc_ndcg_at_5_diff1": 0.41604095040758654, + "nauc_ndcg_at_5_max": 0.4249043802134648, + "nauc_ndcg_at_5_std": 0.17993724939668213, + "nauc_precision_at_1000_diff1": 0.083792013131001, + "nauc_precision_at_1000_max": 0.11941541566268123, + "nauc_precision_at_1000_std": 0.14998003125842968, + "nauc_precision_at_100_diff1": 0.18706431666474574, + "nauc_precision_at_100_max": 0.27483430212725496, + "nauc_precision_at_100_std": 0.2635385039235875, + "nauc_precision_at_10_diff1": 0.3402609243839506, + "nauc_precision_at_10_max": 0.39696290684056373, + "nauc_precision_at_10_std": 0.2543040929122827, + "nauc_precision_at_1_diff1": 0.49148043282667364, + "nauc_precision_at_1_max": 0.4162281278183343, + "nauc_precision_at_1_std": 0.13353245801794972, + "nauc_precision_at_20_diff1": 0.28339601650427404, + "nauc_precision_at_20_max": 0.37034812762645325, + "nauc_precision_at_20_std": 0.28066694878517995, + "nauc_precision_at_3_diff1": 0.42321400240248247, + "nauc_precision_at_3_max": 0.45922334420195193, + "nauc_precision_at_3_std": 0.21665901900964327, + "nauc_precision_at_5_diff1": 0.35882502361537305, + "nauc_precision_at_5_max": 0.4116032274068009, + "nauc_precision_at_5_std": 0.21848845268125894, + "nauc_recall_at_1000_diff1": 0.17469025046598305, + "nauc_recall_at_1000_max": 0.3895257741761885, + "nauc_recall_at_1000_std": 0.3692801971720048, + "nauc_recall_at_100_diff1": 0.2418626980456545, + "nauc_recall_at_100_max": 0.41079255113177915, + "nauc_recall_at_100_std": 0.323352420420411, + "nauc_recall_at_10_diff1": 0.3197249068590067, + "nauc_recall_at_10_max": 0.38848590698639246, + "nauc_recall_at_10_std": 0.21914410672561052, + "nauc_recall_at_1_diff1": 0.49479342920699904, + "nauc_recall_at_1_max": 0.4328222440912369, + "nauc_recall_at_1_std": 0.14008045475803277, + "nauc_recall_at_20_diff1": 0.2830459558937478, + "nauc_recall_at_20_max": 0.3914955914987894, + "nauc_recall_at_20_std": 0.2521367688597883, + "nauc_recall_at_3_diff1": 0.4130246100814953, + "nauc_recall_at_3_max": 0.443396166805615, + "nauc_recall_at_3_std": 0.1981485329094864, + "nauc_recall_at_5_diff1": 0.35208585066200154, + "nauc_recall_at_5_max": 0.40587952768065577, + "nauc_recall_at_5_std": 0.19865152424913415, + "ndcg_at_1": 0.15796, + "ndcg_at_10": 0.22017, + "ndcg_at_100": 0.27021, + "ndcg_at_1000": 0.30016, + "ndcg_at_20": 0.23738, + "ndcg_at_3": 0.18356, + "ndcg_at_5": 0.2005, + "precision_at_1": 0.15796, + "precision_at_10": 0.04017, + "precision_at_100": 0.00764, + "precision_at_1000": 0.00114, + "precision_at_20": 0.02475, + "precision_at_3": 0.08748, + "precision_at_5": 0.06393, + "recall_at_1": 0.12558, + "recall_at_10": 0.3082, + "recall_at_100": 0.52842, + "recall_at_1000": 0.74423, + "recall_at_20": 0.3699, + "recall_at_3": 0.20629, + "recall_at_5": 0.24944 + } + ] + }, + "task_name": "CQADupstackMathematicaRetrieval" +} \ No newline at end of file diff --git a/results/bm25s/0_1_10/CQADupstackPhysicsRetrieval.json b/results/bm25s/0_1_10/CQADupstackPhysicsRetrieval.json new file mode 100644 index 0000000000..85b8275f16 --- /dev/null +++ b/results/bm25s/0_1_10/CQADupstackPhysicsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4", + "evaluation_time": 7.48164439201355, + "kg_co2_emissions": null, + "mteb_version": "1.12.22", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.34926, + "map_at_1": 0.22994, + "map_at_10": 0.3033, + "map_at_100": 0.31339, + "map_at_1000": 0.31467, + "map_at_20": 0.30842, + "map_at_3": 0.28187, + "map_at_5": 0.29388, + "mrr_at_1": 0.2762271414821944, + "mrr_at_10": 0.3516407718043904, + "mrr_at_100": 0.3599429609045099, + "mrr_at_1000": 0.3606546770742375, + "mrr_at_20": 0.35618074020052404, + "mrr_at_3": 0.33156881616939354, + "mrr_at_5": 0.34374398460057726, + "nauc_map_at_1000_diff1": 0.46949749244940286, + "nauc_map_at_1000_max": 0.3670558200582162, + "nauc_map_at_1000_std": 0.08117430402886364, + "nauc_map_at_100_diff1": 0.4691824503083372, + "nauc_map_at_100_max": 0.36695364520253543, + "nauc_map_at_100_std": 0.08110867746277411, + "nauc_map_at_10_diff1": 0.47221443168204646, + "nauc_map_at_10_max": 0.36365870306549647, + "nauc_map_at_10_std": 0.07504532106043542, + "nauc_map_at_1_diff1": 0.520219898804708, + "nauc_map_at_1_max": 0.3632162247591089, + "nauc_map_at_1_std": 0.06283142763330461, + "nauc_map_at_20_diff1": 0.46887851767868377, + "nauc_map_at_20_max": 0.3649838447140216, + "nauc_map_at_20_std": 0.07706656634619133, + "nauc_map_at_3_diff1": 0.4816871837038387, + "nauc_map_at_3_max": 0.3598680539369988, + "nauc_map_at_3_std": 0.06721793673940721, + "nauc_map_at_5_diff1": 0.478755404972199, + "nauc_map_at_5_max": 0.36744591839871604, + "nauc_map_at_5_std": 0.07530177457623627, + "nauc_mrr_at_1000_diff1": 0.4659360976037266, + "nauc_mrr_at_1000_max": 0.37042979249324404, + "nauc_mrr_at_1000_std": 0.08562170764551633, + "nauc_mrr_at_100_diff1": 0.46564190986499165, + "nauc_mrr_at_100_max": 0.3704978849814062, + "nauc_mrr_at_100_std": 0.08575684820994185, + "nauc_mrr_at_10_diff1": 0.46818984448782625, + "nauc_mrr_at_10_max": 0.36913125475529424, + "nauc_mrr_at_10_std": 0.08295354185117973, + "nauc_mrr_at_1_diff1": 0.5136640616948251, + "nauc_mrr_at_1_max": 0.3750235098615363, + "nauc_mrr_at_1_std": 0.07839690674103471, + "nauc_mrr_at_20_diff1": 0.46533584236890335, + "nauc_mrr_at_20_max": 0.3699878315364852, + "nauc_mrr_at_20_std": 0.08372089567050289, + "nauc_mrr_at_3_diff1": 0.472125783771454, + "nauc_mrr_at_3_max": 0.36487311362668867, + "nauc_mrr_at_3_std": 0.07419053535190953, + "nauc_mrr_at_5_diff1": 0.4706494537416062, + "nauc_mrr_at_5_max": 0.37280498445052634, + "nauc_mrr_at_5_std": 0.08275167808435221, + "nauc_ndcg_at_1000_diff1": 0.44672694910632915, + "nauc_ndcg_at_1000_max": 0.37182897067230636, + "nauc_ndcg_at_1000_std": 0.10524955679641716, + "nauc_ndcg_at_100_diff1": 0.43953850631693053, + "nauc_ndcg_at_100_max": 0.3706747781697764, + "nauc_ndcg_at_100_std": 0.10596731236616873, + "nauc_ndcg_at_10_diff1": 0.452963147969717, + "nauc_ndcg_at_10_max": 0.36177982633459177, + "nauc_ndcg_at_10_std": 0.08022386206885859, + "nauc_ndcg_at_1_diff1": 0.5136640616948251, + "nauc_ndcg_at_1_max": 0.3750235098615363, + "nauc_ndcg_at_1_std": 0.07839690674103471, + "nauc_ndcg_at_20_diff1": 0.4410524931649828, + "nauc_ndcg_at_20_max": 0.36539736522926936, + "nauc_ndcg_at_20_std": 0.08634255457207066, + "nauc_ndcg_at_3_diff1": 0.46725353660447655, + "nauc_ndcg_at_3_max": 0.36038638082133423, + "nauc_ndcg_at_3_std": 0.06860682458347885, + "nauc_ndcg_at_5_diff1": 0.46487605796461584, + "nauc_ndcg_at_5_max": 0.37256978251290795, + "nauc_ndcg_at_5_std": 0.08217757600180622, + "nauc_precision_at_1000_diff1": -0.042157404546289044, + "nauc_precision_at_1000_max": 0.03094027530391, + "nauc_precision_at_1000_std": 0.06974549327281317, + "nauc_precision_at_100_diff1": 0.07872007613338024, + "nauc_precision_at_100_max": 0.19094998603108493, + "nauc_precision_at_100_std": 0.14819952961452695, + "nauc_precision_at_10_diff1": 0.27032817588788255, + "nauc_precision_at_10_max": 0.29294467710299393, + "nauc_precision_at_10_std": 0.1033495388879284, + "nauc_precision_at_1_diff1": 0.5136640616948251, + "nauc_precision_at_1_max": 0.3750235098615363, + "nauc_precision_at_1_std": 0.07839690674103471, + "nauc_precision_at_20_diff1": 0.18910429422408154, + "nauc_precision_at_20_max": 0.2696183098190711, + "nauc_precision_at_20_std": 0.1116988262179429, + "nauc_precision_at_3_diff1": 0.3760560120508215, + "nauc_precision_at_3_max": 0.33288547777402616, + "nauc_precision_at_3_std": 0.07841439122865931, + "nauc_precision_at_5_diff1": 0.34669997392171825, + "nauc_precision_at_5_max": 0.34351234067715686, + "nauc_precision_at_5_std": 0.10454593115122861, + "nauc_recall_at_1000_diff1": 0.27858986328773233, + "nauc_recall_at_1000_max": 0.3428739394798069, + "nauc_recall_at_1000_std": 0.2861253673047428, + "nauc_recall_at_100_diff1": 0.28437896666543555, + "nauc_recall_at_100_max": 0.33068791716818063, + "nauc_recall_at_100_std": 0.19540043839377608, + "nauc_recall_at_10_diff1": 0.3803897426128292, + "nauc_recall_at_10_max": 0.3271011953342343, + "nauc_recall_at_10_std": 0.08037261487526944, + "nauc_recall_at_1_diff1": 0.520219898804708, + "nauc_recall_at_1_max": 0.3632162247591089, + "nauc_recall_at_1_std": 0.06283142763330461, + "nauc_recall_at_20_diff1": 0.328305567663769, + "nauc_recall_at_20_max": 0.32814871997927, + "nauc_recall_at_20_std": 0.09772741214804276, + "nauc_recall_at_3_diff1": 0.4378396747889548, + "nauc_recall_at_3_max": 0.34226672731790897, + "nauc_recall_at_3_std": 0.05993464213546822, + "nauc_recall_at_5_diff1": 0.4200085799836254, + "nauc_recall_at_5_max": 0.36300150765781514, + "nauc_recall_at_5_std": 0.08815833994198002, + "ndcg_at_1": 0.27623, + "ndcg_at_10": 0.34926, + "ndcg_at_100": 0.39834, + "ndcg_at_1000": 0.42702, + "ndcg_at_20": 0.36598, + "ndcg_at_3": 0.31344, + "ndcg_at_5": 0.3307, + "precision_at_1": 0.27623, + "precision_at_10": 0.06006, + "precision_at_100": 0.00988, + "precision_at_1000": 0.00141, + "precision_at_20": 0.03542, + "precision_at_3": 0.14501, + "precision_at_5": 0.10125, + "recall_at_1": 0.22994, + "recall_at_10": 0.44048, + "recall_at_100": 0.65521, + "recall_at_1000": 0.85317, + "recall_at_20": 0.49913, + "recall_at_3": 0.3366, + "recall_at_5": 0.38253 + } + ] + }, + "task_name": "CQADupstackPhysicsRetrieval" +} \ No newline at end of file diff --git a/results/bm25s/0_1_10/CQADupstackProgrammersRetrieval.json b/results/bm25s/0_1_10/CQADupstackProgrammersRetrieval.json new file mode 100644 index 0000000000..e8f6d8fd2e --- /dev/null +++ b/results/bm25s/0_1_10/CQADupstackProgrammersRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6184bc1440d2dbc7612be22b50686b8826d22b32", + "evaluation_time": 7.248922348022461, + "kg_co2_emissions": null, + "mteb_version": "1.12.22", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.298, + "map_at_1": 0.1985, + "map_at_10": 0.26064, + "map_at_100": 0.27102, + "map_at_1000": 0.27222, + "map_at_20": 0.26611, + "map_at_3": 0.24178, + "map_at_5": 0.25266, + "mrr_at_1": 0.23858447488584475, + "mrr_at_10": 0.298684496629702, + "mrr_at_100": 0.3075363417209715, + "mrr_at_1000": 0.30828827921046104, + "mrr_at_20": 0.3038291701059682, + "mrr_at_3": 0.2804414003044141, + "mrr_at_5": 0.2911149162861492, + "nauc_map_at_1000_diff1": 0.4754604358607166, + "nauc_map_at_1000_max": 0.40270088100620205, + "nauc_map_at_1000_std": 0.12616614847762833, + "nauc_map_at_100_diff1": 0.47554949718231593, + "nauc_map_at_100_max": 0.40270449018228294, + "nauc_map_at_100_std": 0.12598014572428878, + "nauc_map_at_10_diff1": 0.4806599833546847, + "nauc_map_at_10_max": 0.403876216930464, + "nauc_map_at_10_std": 0.12015770078239815, + "nauc_map_at_1_diff1": 0.5318608320992171, + "nauc_map_at_1_max": 0.4210559799721134, + "nauc_map_at_1_std": 0.11596637824034386, + "nauc_map_at_20_diff1": 0.4759938509391016, + "nauc_map_at_20_max": 0.402331262554285, + "nauc_map_at_20_std": 0.12280783418237597, + "nauc_map_at_3_diff1": 0.49258810912150924, + "nauc_map_at_3_max": 0.4085870372622373, + "nauc_map_at_3_std": 0.11098105345229789, + "nauc_map_at_5_diff1": 0.4843441459226057, + "nauc_map_at_5_max": 0.40498520238714086, + "nauc_map_at_5_std": 0.11812184513432976, + "nauc_mrr_at_1000_diff1": 0.4806684257184316, + "nauc_mrr_at_1000_max": 0.4138397232060037, + "nauc_mrr_at_1000_std": 0.14258547968788915, + "nauc_mrr_at_100_diff1": 0.4804940352732612, + "nauc_mrr_at_100_max": 0.4138691111683882, + "nauc_mrr_at_100_std": 0.14275794818073878, + "nauc_mrr_at_10_diff1": 0.48379738740954775, + "nauc_mrr_at_10_max": 0.41394709192925183, + "nauc_mrr_at_10_std": 0.13825018093746314, + "nauc_mrr_at_1_diff1": 0.5438751103239798, + "nauc_mrr_at_1_max": 0.45456270918942443, + "nauc_mrr_at_1_std": 0.1567127804721244, + "nauc_mrr_at_20_diff1": 0.4802657428390989, + "nauc_mrr_at_20_max": 0.413047429221462, + "nauc_mrr_at_20_std": 0.14071035862016762, + "nauc_mrr_at_3_diff1": 0.4954639339001014, + "nauc_mrr_at_3_max": 0.4217066371816253, + "nauc_mrr_at_3_std": 0.13604299676148468, + "nauc_mrr_at_5_diff1": 0.48633869010791697, + "nauc_mrr_at_5_max": 0.4142073173009765, + "nauc_mrr_at_5_std": 0.13581892085708452, + "nauc_ndcg_at_1000_diff1": 0.4424890337762341, + "nauc_ndcg_at_1000_max": 0.3948936151741808, + "nauc_ndcg_at_1000_std": 0.1552011070506116, + "nauc_ndcg_at_100_diff1": 0.4414358029294326, + "nauc_ndcg_at_100_max": 0.3947451147872283, + "nauc_ndcg_at_100_std": 0.1574821563817625, + "nauc_ndcg_at_10_diff1": 0.4568084096391255, + "nauc_ndcg_at_10_max": 0.39325384400610713, + "nauc_ndcg_at_10_std": 0.1256257622981702, + "nauc_ndcg_at_1_diff1": 0.5438751103239798, + "nauc_ndcg_at_1_max": 0.45456270918942443, + "nauc_ndcg_at_1_std": 0.1567127804721244, + "nauc_ndcg_at_20_diff1": 0.4430589505005143, + "nauc_ndcg_at_20_max": 0.3889638976896975, + "nauc_ndcg_at_20_std": 0.13460593127072237, + "nauc_ndcg_at_3_diff1": 0.4770816582731393, + "nauc_ndcg_at_3_max": 0.40487999672633845, + "nauc_ndcg_at_3_std": 0.11769789118788737, + "nauc_ndcg_at_5_diff1": 0.4635004722354713, + "nauc_ndcg_at_5_max": 0.394432974233501, + "nauc_ndcg_at_5_std": 0.12021252882049803, + "nauc_precision_at_1000_diff1": -0.024178352995859662, + "nauc_precision_at_1000_max": 0.03355243147461784, + "nauc_precision_at_1000_std": 0.09589478315179775, + "nauc_precision_at_100_diff1": 0.11133979348963108, + "nauc_precision_at_100_max": 0.1857600278615175, + "nauc_precision_at_100_std": 0.20371083595189943, + "nauc_precision_at_10_diff1": 0.30597446666527606, + "nauc_precision_at_10_max": 0.30390701847449814, + "nauc_precision_at_10_std": 0.14853373598224204, + "nauc_precision_at_1_diff1": 0.5438751103239798, + "nauc_precision_at_1_max": 0.45456270918942443, + "nauc_precision_at_1_std": 0.1567127804721244, + "nauc_precision_at_20_diff1": 0.22356828129547932, + "nauc_precision_at_20_max": 0.264627400489263, + "nauc_precision_at_20_std": 0.17545735286563421, + "nauc_precision_at_3_diff1": 0.4261101426424948, + "nauc_precision_at_3_max": 0.38983715037660205, + "nauc_precision_at_3_std": 0.13282969180069706, + "nauc_precision_at_5_diff1": 0.3579849547285407, + "nauc_precision_at_5_max": 0.34104400207350516, + "nauc_precision_at_5_std": 0.14316544918836632, + "nauc_recall_at_1000_diff1": 0.25117064604731565, + "nauc_recall_at_1000_max": 0.34247066671734006, + "nauc_recall_at_1000_std": 0.3242774284321667, + "nauc_recall_at_100_diff1": 0.30049590115645125, + "nauc_recall_at_100_max": 0.351014019151641, + "nauc_recall_at_100_std": 0.2779052945176138, + "nauc_recall_at_10_diff1": 0.37750383943840066, + "nauc_recall_at_10_max": 0.3493604987007066, + "nauc_recall_at_10_std": 0.12599375978882263, + "nauc_recall_at_1_diff1": 0.5318608320992171, + "nauc_recall_at_1_max": 0.4210559799721134, + "nauc_recall_at_1_std": 0.11596637824034386, + "nauc_recall_at_20_diff1": 0.32603694535336614, + "nauc_recall_at_20_max": 0.32821136033289783, + "nauc_recall_at_20_std": 0.1517969917811495, + "nauc_recall_at_3_diff1": 0.43913966831786555, + "nauc_recall_at_3_max": 0.3776260735074705, + "nauc_recall_at_3_std": 0.09981855768052304, + "nauc_recall_at_5_diff1": 0.40380104533139527, + "nauc_recall_at_5_max": 0.35493563297090036, + "nauc_recall_at_5_std": 0.10934097207195613, + "ndcg_at_1": 0.23858, + "ndcg_at_10": 0.298, + "ndcg_at_100": 0.34894, + "ndcg_at_1000": 0.3771, + "ndcg_at_20": 0.31616, + "ndcg_at_3": 0.26548, + "ndcg_at_5": 0.28073, + "precision_at_1": 0.23858, + "precision_at_10": 0.05205, + "precision_at_100": 0.00904, + "precision_at_1000": 0.00131, + "precision_at_20": 0.03139, + "precision_at_3": 0.12253, + "precision_at_5": 0.08653, + "recall_at_1": 0.1985, + "recall_at_10": 0.37201, + "recall_at_100": 0.59912, + "recall_at_1000": 0.79744, + "recall_at_20": 0.43641, + "recall_at_3": 0.28249, + "recall_at_5": 0.32184 + } + ] + }, + "task_name": "CQADupstackProgrammersRetrieval" +} \ No newline at end of file diff --git a/results/bm25s/0_1_10/CQADupstackStatsRetrieval.json b/results/bm25s/0_1_10/CQADupstackStatsRetrieval.json new file mode 100644 index 0000000000..62fdbe9171 --- /dev/null +++ b/results/bm25s/0_1_10/CQADupstackStatsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "65ac3a16b8e91f9cee4c9828cc7c335575432a2a", + "evaluation_time": 8.498745918273926, + "kg_co2_emissions": null, + "mteb_version": "1.12.22", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.29019, + "map_at_1": 0.20576, + "map_at_10": 0.25749, + "map_at_100": 0.26567, + "map_at_1000": 0.26658, + "map_at_20": 0.26172, + "map_at_3": 0.24163, + "map_at_5": 0.24995, + "mrr_at_1": 0.22699386503067484, + "mrr_at_10": 0.27969312980816047, + "mrr_at_100": 0.2876246633455716, + "mrr_at_1000": 0.28834994653050755, + "mrr_at_20": 0.2841803387061845, + "mrr_at_3": 0.2632924335378325, + "mrr_at_5": 0.27134458077709617, + "nauc_map_at_1000_diff1": 0.49182526804988347, + "nauc_map_at_1000_max": 0.36873872142012515, + "nauc_map_at_1000_std": 0.14748881004508416, + "nauc_map_at_100_diff1": 0.49188835278369475, + "nauc_map_at_100_max": 0.3685496628618809, + "nauc_map_at_100_std": 0.1470562433855383, + "nauc_map_at_10_diff1": 0.4940061579990735, + "nauc_map_at_10_max": 0.3676674340458906, + "nauc_map_at_10_std": 0.1434876454944314, + "nauc_map_at_1_diff1": 0.5626271531576055, + "nauc_map_at_1_max": 0.3833771816183858, + "nauc_map_at_1_std": 0.12263638852712834, + "nauc_map_at_20_diff1": 0.493734183724005, + "nauc_map_at_20_max": 0.36787977076815714, + "nauc_map_at_20_std": 0.14449220514670694, + "nauc_map_at_3_diff1": 0.4988778004678863, + "nauc_map_at_3_max": 0.3688424752335865, + "nauc_map_at_3_std": 0.14157143843465925, + "nauc_map_at_5_diff1": 0.5021232754446784, + "nauc_map_at_5_max": 0.3712264681172258, + "nauc_map_at_5_std": 0.14476672609617086, + "nauc_mrr_at_1000_diff1": 0.49940904783445583, + "nauc_mrr_at_1000_max": 0.38522481112710516, + "nauc_mrr_at_1000_std": 0.15600780868326086, + "nauc_mrr_at_100_diff1": 0.499421876272215, + "nauc_mrr_at_100_max": 0.3850866335776641, + "nauc_mrr_at_100_std": 0.15559072517187453, + "nauc_mrr_at_10_diff1": 0.5013088531037604, + "nauc_mrr_at_10_max": 0.3859334056074047, + "nauc_mrr_at_10_std": 0.15402831215184265, + "nauc_mrr_at_1_diff1": 0.5671897425018518, + "nauc_mrr_at_1_max": 0.4020407800772307, + "nauc_mrr_at_1_std": 0.13813549524888932, + "nauc_mrr_at_20_diff1": 0.5011557430375656, + "nauc_mrr_at_20_max": 0.3853227208528792, + "nauc_mrr_at_20_std": 0.15410755142160923, + "nauc_mrr_at_3_diff1": 0.5044843791989136, + "nauc_mrr_at_3_max": 0.38712077697824226, + "nauc_mrr_at_3_std": 0.15190130809142757, + "nauc_mrr_at_5_diff1": 0.5109797903351286, + "nauc_mrr_at_5_max": 0.39001286529911977, + "nauc_mrr_at_5_std": 0.15456209527824694, + "nauc_ndcg_at_1000_diff1": 0.4565956005210591, + "nauc_ndcg_at_1000_max": 0.36581727293764177, + "nauc_ndcg_at_1000_std": 0.17491454421484157, + "nauc_ndcg_at_100_diff1": 0.45555881647727303, + "nauc_ndcg_at_100_max": 0.3596412862063693, + "nauc_ndcg_at_100_std": 0.1646364741455819, + "nauc_ndcg_at_10_diff1": 0.4693237118568356, + "nauc_ndcg_at_10_max": 0.36205761558135763, + "nauc_ndcg_at_10_std": 0.14978348648726006, + "nauc_ndcg_at_1_diff1": 0.5671897425018518, + "nauc_ndcg_at_1_max": 0.4020407800772307, + "nauc_ndcg_at_1_std": 0.13813549524888932, + "nauc_ndcg_at_20_diff1": 0.4672353307674374, + "nauc_ndcg_at_20_max": 0.3593307149100604, + "nauc_ndcg_at_20_std": 0.15050390258231658, + "nauc_ndcg_at_3_diff1": 0.47786169361038633, + "nauc_ndcg_at_3_max": 0.36675457431278624, + "nauc_ndcg_at_3_std": 0.1468621332854214, + "nauc_ndcg_at_5_diff1": 0.4874421849305086, + "nauc_ndcg_at_5_max": 0.37105790296381674, + "nauc_ndcg_at_5_std": 0.1527746297320774, + "nauc_precision_at_1000_diff1": 0.04090883239946926, + "nauc_precision_at_1000_max": 0.19437583151606772, + "nauc_precision_at_1000_std": 0.20389294163538563, + "nauc_precision_at_100_diff1": 0.1876627668375524, + "nauc_precision_at_100_max": 0.2578167943780651, + "nauc_precision_at_100_std": 0.19508352864109332, + "nauc_precision_at_10_diff1": 0.3397939986414335, + "nauc_precision_at_10_max": 0.3256265372938673, + "nauc_precision_at_10_std": 0.15279848300746396, + "nauc_precision_at_1_diff1": 0.5671897425018518, + "nauc_precision_at_1_max": 0.4020407800772307, + "nauc_precision_at_1_std": 0.13813549524888932, + "nauc_precision_at_20_diff1": 0.3254215291869494, + "nauc_precision_at_20_max": 0.31405464884777995, + "nauc_precision_at_20_std": 0.15325729869805313, + "nauc_precision_at_3_diff1": 0.3943499811946167, + "nauc_precision_at_3_max": 0.3490252302750242, + "nauc_precision_at_3_std": 0.15539591839363046, + "nauc_precision_at_5_diff1": 0.39857898332554426, + "nauc_precision_at_5_max": 0.34756890612226965, + "nauc_precision_at_5_std": 0.15793971098858453, + "nauc_recall_at_1000_diff1": 0.259425946896802, + "nauc_recall_at_1000_max": 0.3121168808408477, + "nauc_recall_at_1000_std": 0.33156078587466514, + "nauc_recall_at_100_diff1": 0.3055174368667267, + "nauc_recall_at_100_max": 0.2870397743485396, + "nauc_recall_at_100_std": 0.21394461026621203, + "nauc_recall_at_10_diff1": 0.38842072811112227, + "nauc_recall_at_10_max": 0.3223999510402151, + "nauc_recall_at_10_std": 0.15358115838818498, + "nauc_recall_at_1_diff1": 0.5626271531576055, + "nauc_recall_at_1_max": 0.3833771816183858, + "nauc_recall_at_1_std": 0.12263638852712834, + "nauc_recall_at_20_diff1": 0.3705686173258057, + "nauc_recall_at_20_max": 0.3001105855119193, + "nauc_recall_at_20_std": 0.1496546320941287, + "nauc_recall_at_3_diff1": 0.42706416826354054, + "nauc_recall_at_3_max": 0.34545806521609934, + "nauc_recall_at_3_std": 0.15692661691639442, + "nauc_recall_at_5_diff1": 0.4378703372980789, + "nauc_recall_at_5_max": 0.3465498455620958, + "nauc_recall_at_5_std": 0.16383812778724569, + "ndcg_at_1": 0.22699, + "ndcg_at_10": 0.29019, + "ndcg_at_100": 0.33283, + "ndcg_at_1000": 0.35824, + "ndcg_at_20": 0.30604, + "ndcg_at_3": 0.25962, + "ndcg_at_5": 0.27257, + "precision_at_1": 0.22699, + "precision_at_10": 0.04387, + "precision_at_100": 0.00693, + "precision_at_1000": 0.001, + "precision_at_20": 0.02546, + "precision_at_3": 0.10941, + "precision_at_5": 0.07393, + "recall_at_1": 0.20576, + "recall_at_10": 0.36976, + "recall_at_100": 0.56734, + "recall_at_1000": 0.75664, + "recall_at_20": 0.43183, + "recall_at_3": 0.28337, + "recall_at_5": 0.3171 + } + ] + }, + "task_name": "CQADupstackStatsRetrieval" +} \ No newline at end of file diff --git a/results/bm25s/0_1_10/CQADupstackTexRetrieval.json b/results/bm25s/0_1_10/CQADupstackTexRetrieval.json new file mode 100644 index 0000000000..637be559a5 --- /dev/null +++ b/results/bm25s/0_1_10/CQADupstackTexRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "46989137a86843e03a6195de44b09deda022eec7", + "evaluation_time": 21.11893892288208, + "kg_co2_emissions": null, + "mteb_version": "1.12.22", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.24397, + "map_at_1": 0.16121, + "map_at_10": 0.21155, + "map_at_100": 0.21954, + "map_at_1000": 0.22067, + "map_at_20": 0.2156, + "map_at_3": 0.19585, + "map_at_5": 0.20534, + "mrr_at_1": 0.1930488644184446, + "mrr_at_10": 0.2431616578726007, + "mrr_at_100": 0.2508779929410238, + "mrr_at_1000": 0.25171349736925525, + "mrr_at_20": 0.24713996499748828, + "mrr_at_3": 0.22740307409956415, + "mrr_at_5": 0.23664257857306736, + "nauc_map_at_1000_diff1": 0.4577695411464286, + "nauc_map_at_1000_max": 0.3818562008864924, + "nauc_map_at_1000_std": 0.14749226791364445, + "nauc_map_at_100_diff1": 0.4582039069989363, + "nauc_map_at_100_max": 0.38175390494808337, + "nauc_map_at_100_std": 0.14719387973379264, + "nauc_map_at_10_diff1": 0.46265432212057295, + "nauc_map_at_10_max": 0.3828380379984688, + "nauc_map_at_10_std": 0.14293208695403636, + "nauc_map_at_1_diff1": 0.5070707014940492, + "nauc_map_at_1_max": 0.37324718407668506, + "nauc_map_at_1_std": 0.12718661723665325, + "nauc_map_at_20_diff1": 0.4597954747680448, + "nauc_map_at_20_max": 0.3814851536455852, + "nauc_map_at_20_std": 0.14497713497302916, + "nauc_map_at_3_diff1": 0.47373722266976775, + "nauc_map_at_3_max": 0.3840806846755847, + "nauc_map_at_3_std": 0.13633343287216704, + "nauc_map_at_5_diff1": 0.4689449090412826, + "nauc_map_at_5_max": 0.38625057322947554, + "nauc_map_at_5_std": 0.14224106334102474, + "nauc_mrr_at_1000_diff1": 0.45261376444766527, + "nauc_mrr_at_1000_max": 0.38656552109949965, + "nauc_mrr_at_1000_std": 0.14618099301460913, + "nauc_mrr_at_100_diff1": 0.452685966454595, + "nauc_mrr_at_100_max": 0.38643356899101283, + "nauc_mrr_at_100_std": 0.14598319194473122, + "nauc_mrr_at_10_diff1": 0.45610763145449723, + "nauc_mrr_at_10_max": 0.387863100789747, + "nauc_mrr_at_10_std": 0.14295693213083766, + "nauc_mrr_at_1_diff1": 0.5010804707372724, + "nauc_mrr_at_1_max": 0.38774411826813365, + "nauc_mrr_at_1_std": 0.1324195022222361, + "nauc_mrr_at_20_diff1": 0.45383922243618385, + "nauc_mrr_at_20_max": 0.38611182577651376, + "nauc_mrr_at_20_std": 0.14408603604336248, + "nauc_mrr_at_3_diff1": 0.46987336315675815, + "nauc_mrr_at_3_max": 0.39234013392005723, + "nauc_mrr_at_3_std": 0.13906144854879612, + "nauc_mrr_at_5_diff1": 0.4633901881355734, + "nauc_mrr_at_5_max": 0.39171460773169364, + "nauc_mrr_at_5_std": 0.14289959734043994, + "nauc_ndcg_at_1000_diff1": 0.4152555321511589, + "nauc_ndcg_at_1000_max": 0.3796264326339578, + "nauc_ndcg_at_1000_std": 0.17435262703779264, + "nauc_ndcg_at_100_diff1": 0.42030745301500677, + "nauc_ndcg_at_100_max": 0.37614661167639374, + "nauc_ndcg_at_100_std": 0.16822980723767947, + "nauc_ndcg_at_10_diff1": 0.4408382220945322, + "nauc_ndcg_at_10_max": 0.3809174139515055, + "nauc_ndcg_at_10_std": 0.1484718795043987, + "nauc_ndcg_at_1_diff1": 0.5010804707372724, + "nauc_ndcg_at_1_max": 0.38774411826813365, + "nauc_ndcg_at_1_std": 0.1324195022222361, + "nauc_ndcg_at_20_diff1": 0.43162946050832446, + "nauc_ndcg_at_20_max": 0.3755873278571746, + "nauc_ndcg_at_20_std": 0.15451539634873981, + "nauc_ndcg_at_3_diff1": 0.46432922032346907, + "nauc_ndcg_at_3_max": 0.38827861990062096, + "nauc_ndcg_at_3_std": 0.13838330603179505, + "nauc_ndcg_at_5_diff1": 0.45563697436268186, + "nauc_ndcg_at_5_max": 0.38951625098696513, + "nauc_ndcg_at_5_std": 0.14721085448661705, + "nauc_precision_at_1000_diff1": -0.013475314433183113, + "nauc_precision_at_1000_max": 0.10136271831673066, + "nauc_precision_at_1000_std": 0.07785469095751375, + "nauc_precision_at_100_diff1": 0.16895651617412072, + "nauc_precision_at_100_max": 0.263496004436288, + "nauc_precision_at_100_std": 0.18071613548272106, + "nauc_precision_at_10_diff1": 0.3413218286241948, + "nauc_precision_at_10_max": 0.363279193066599, + "nauc_precision_at_10_std": 0.16309510751417727, + "nauc_precision_at_1_diff1": 0.5010804707372724, + "nauc_precision_at_1_max": 0.38774411826813365, + "nauc_precision_at_1_std": 0.1324195022222361, + "nauc_precision_at_20_diff1": 0.2891715406054205, + "nauc_precision_at_20_max": 0.32646022876712766, + "nauc_precision_at_20_std": 0.16779196928961035, + "nauc_precision_at_3_diff1": 0.42366466932757973, + "nauc_precision_at_3_max": 0.3956558144487581, + "nauc_precision_at_3_std": 0.142641716737621, + "nauc_precision_at_5_diff1": 0.3940081371018236, + "nauc_precision_at_5_max": 0.3921836099899726, + "nauc_precision_at_5_std": 0.1584928254080098, + "nauc_recall_at_1000_diff1": 0.1997054274646303, + "nauc_recall_at_1000_max": 0.33294822702385, + "nauc_recall_at_1000_std": 0.30570811708673307, + "nauc_recall_at_100_diff1": 0.28280483510212895, + "nauc_recall_at_100_max": 0.3226723971584298, + "nauc_recall_at_100_std": 0.2277481773714978, + "nauc_recall_at_10_diff1": 0.3686189075749225, + "nauc_recall_at_10_max": 0.3471332008234341, + "nauc_recall_at_10_std": 0.1521624530307365, + "nauc_recall_at_1_diff1": 0.5070707014940492, + "nauc_recall_at_1_max": 0.37324718407668506, + "nauc_recall_at_1_std": 0.12718661723665325, + "nauc_recall_at_20_diff1": 0.33976788493431137, + "nauc_recall_at_20_max": 0.3287471202107221, + "nauc_recall_at_20_std": 0.17002332490434344, + "nauc_recall_at_3_diff1": 0.42819789782458095, + "nauc_recall_at_3_max": 0.37312432410183277, + "nauc_recall_at_3_std": 0.13973631709494022, + "nauc_recall_at_5_diff1": 0.4131530040219873, + "nauc_recall_at_5_max": 0.37789656696793755, + "nauc_recall_at_5_std": 0.15667566944660108, + "ndcg_at_1": 0.19305, + "ndcg_at_10": 0.24397, + "ndcg_at_100": 0.28517, + "ndcg_at_1000": 0.31507, + "ndcg_at_20": 0.25785, + "ndcg_at_3": 0.21563, + "ndcg_at_5": 0.22986, + "precision_at_1": 0.19305, + "precision_at_10": 0.04085, + "precision_at_100": 0.00704, + "precision_at_1000": 0.00112, + "precision_at_20": 0.02431, + "precision_at_3": 0.09738, + "precision_at_5": 0.06903, + "recall_at_1": 0.16121, + "recall_at_10": 0.3109, + "recall_at_100": 0.49977, + "recall_at_1000": 0.71775, + "recall_at_20": 0.36305, + "recall_at_3": 0.23182, + "recall_at_5": 0.26847 + } + ] + }, + "task_name": "CQADupstackTexRetrieval" +} \ No newline at end of file diff --git a/results/bm25s/0_1_10/CQADupstackUnixRetrieval.json b/results/bm25s/0_1_10/CQADupstackUnixRetrieval.json new file mode 100644 index 0000000000..9123bb5392 --- /dev/null +++ b/results/bm25s/0_1_10/CQADupstackUnixRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6c6430d3a6d36f8d2a829195bc5dc94d7e063e53", + "evaluation_time": 10.986534833908081, + "kg_co2_emissions": null, + "mteb_version": "1.12.22", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.29165, + "map_at_1": 0.20827, + "map_at_10": 0.25661, + "map_at_100": 0.26501, + "map_at_1000": 0.26607, + "map_at_20": 0.26096, + "map_at_3": 0.24005, + "map_at_5": 0.24761, + "mrr_at_1": 0.23973880597014927, + "mrr_at_10": 0.2883806414356787, + "mrr_at_100": 0.29613330900464807, + "mrr_at_1000": 0.2969648092786831, + "mrr_at_20": 0.2927101006729229, + "mrr_at_3": 0.27036691542288555, + "mrr_at_5": 0.278575870646766, + "nauc_map_at_1000_diff1": 0.5543947735081151, + "nauc_map_at_1000_max": 0.5335527248100502, + "nauc_map_at_1000_std": 0.19226238530546333, + "nauc_map_at_100_diff1": 0.5542122511679982, + "nauc_map_at_100_max": 0.5336140614347408, + "nauc_map_at_100_std": 0.19224276161971893, + "nauc_map_at_10_diff1": 0.5567099238187042, + "nauc_map_at_10_max": 0.5340313168748065, + "nauc_map_at_10_std": 0.1888371520412531, + "nauc_map_at_1_diff1": 0.6313580571991636, + "nauc_map_at_1_max": 0.5458738767477077, + "nauc_map_at_1_std": 0.1811585344737397, + "nauc_map_at_20_diff1": 0.5564665725633545, + "nauc_map_at_20_max": 0.5337010223627309, + "nauc_map_at_20_std": 0.18958292129580984, + "nauc_map_at_3_diff1": 0.5750110008737679, + "nauc_map_at_3_max": 0.5463911852061891, + "nauc_map_at_3_std": 0.18842016220518829, + "nauc_map_at_5_diff1": 0.5689865839588846, + "nauc_map_at_5_max": 0.5423211883542725, + "nauc_map_at_5_std": 0.19193801864694945, + "nauc_mrr_at_1000_diff1": 0.5438838079987897, + "nauc_mrr_at_1000_max": 0.5248561128554384, + "nauc_mrr_at_1000_std": 0.18372484612931012, + "nauc_mrr_at_100_diff1": 0.5436700788398666, + "nauc_mrr_at_100_max": 0.524857077423649, + "nauc_mrr_at_100_std": 0.18375541261566236, + "nauc_mrr_at_10_diff1": 0.5452965981863154, + "nauc_mrr_at_10_max": 0.5251506791107123, + "nauc_mrr_at_10_std": 0.18076712638743964, + "nauc_mrr_at_1_diff1": 0.6163933209900965, + "nauc_mrr_at_1_max": 0.5422084176071401, + "nauc_mrr_at_1_std": 0.1690918470374381, + "nauc_mrr_at_20_diff1": 0.5448482600120742, + "nauc_mrr_at_20_max": 0.525036857570174, + "nauc_mrr_at_20_std": 0.18199242805722315, + "nauc_mrr_at_3_diff1": 0.5651495810312022, + "nauc_mrr_at_3_max": 0.5379486800997548, + "nauc_mrr_at_3_std": 0.18046459471888085, + "nauc_mrr_at_5_diff1": 0.5596414684081534, + "nauc_mrr_at_5_max": 0.5356653862316442, + "nauc_mrr_at_5_std": 0.1850500983231059, + "nauc_ndcg_at_1000_diff1": 0.506225475344358, + "nauc_ndcg_at_1000_max": 0.5125356785145525, + "nauc_ndcg_at_1000_std": 0.2100993823261888, + "nauc_ndcg_at_100_diff1": 0.5019630272920849, + "nauc_ndcg_at_100_max": 0.5121768422579762, + "nauc_ndcg_at_100_std": 0.2090797549231837, + "nauc_ndcg_at_10_diff1": 0.5176428682518881, + "nauc_ndcg_at_10_max": 0.5142988794852442, + "nauc_ndcg_at_10_std": 0.18761298366278856, + "nauc_ndcg_at_1_diff1": 0.6163933209900965, + "nauc_ndcg_at_1_max": 0.5422084176071401, + "nauc_ndcg_at_1_std": 0.1690918470374381, + "nauc_ndcg_at_20_diff1": 0.5161244391636185, + "nauc_ndcg_at_20_max": 0.5129282150515858, + "nauc_ndcg_at_20_std": 0.19070241517577619, + "nauc_ndcg_at_3_diff1": 0.5553148432947544, + "nauc_ndcg_at_3_max": 0.5407151465464106, + "nauc_ndcg_at_3_std": 0.18835583477795448, + "nauc_ndcg_at_5_diff1": 0.5473914724714909, + "nauc_ndcg_at_5_max": 0.5361340829627993, + "nauc_ndcg_at_5_std": 0.19580930231900656, + "nauc_precision_at_1000_diff1": -0.013393763159076586, + "nauc_precision_at_1000_max": 0.07578581320063632, + "nauc_precision_at_1000_std": 0.09264793586394685, + "nauc_precision_at_100_diff1": 0.15727118656689065, + "nauc_precision_at_100_max": 0.2926658114751574, + "nauc_precision_at_100_std": 0.20665654079764145, + "nauc_precision_at_10_diff1": 0.3400747539381544, + "nauc_precision_at_10_max": 0.43255487121329084, + "nauc_precision_at_10_std": 0.17517154441178281, + "nauc_precision_at_1_diff1": 0.6163933209900965, + "nauc_precision_at_1_max": 0.5422084176071401, + "nauc_precision_at_1_std": 0.1690918470374381, + "nauc_precision_at_20_diff1": 0.31063584350778345, + "nauc_precision_at_20_max": 0.39830590747671263, + "nauc_precision_at_20_std": 0.17323675678320566, + "nauc_precision_at_3_diff1": 0.4602907816081777, + "nauc_precision_at_3_max": 0.509653690092353, + "nauc_precision_at_3_std": 0.17531174330337626, + "nauc_precision_at_5_diff1": 0.436530093924796, + "nauc_precision_at_5_max": 0.498375908840078, + "nauc_precision_at_5_std": 0.1953811822345095, + "nauc_recall_at_1000_diff1": 0.2465448832055338, + "nauc_recall_at_1000_max": 0.38283771062380123, + "nauc_recall_at_1000_std": 0.35533486622127236, + "nauc_recall_at_100_diff1": 0.30740008438564337, + "nauc_recall_at_100_max": 0.4162086427729747, + "nauc_recall_at_100_std": 0.2806392031941819, + "nauc_recall_at_10_diff1": 0.4043428458161273, + "nauc_recall_at_10_max": 0.44298638536519863, + "nauc_recall_at_10_std": 0.1796089342611097, + "nauc_recall_at_1_diff1": 0.6313580571991636, + "nauc_recall_at_1_max": 0.5458738767477077, + "nauc_recall_at_1_std": 0.1811585344737397, + "nauc_recall_at_20_diff1": 0.3925681297842023, + "nauc_recall_at_20_max": 0.4316658704470397, + "nauc_recall_at_20_std": 0.18836443190865743, + "nauc_recall_at_3_diff1": 0.5220521838345392, + "nauc_recall_at_3_max": 0.5373536248455767, + "nauc_recall_at_3_std": 0.19790826350147528, + "nauc_recall_at_5_diff1": 0.49076141279009033, + "nauc_recall_at_5_max": 0.5124412756094869, + "nauc_recall_at_5_std": 0.2081330577231065, + "ndcg_at_1": 0.23974, + "ndcg_at_10": 0.29165, + "ndcg_at_100": 0.33495, + "ndcg_at_1000": 0.3643, + "ndcg_at_20": 0.3068, + "ndcg_at_3": 0.25672, + "ndcg_at_5": 0.26966, + "precision_at_1": 0.23974, + "precision_at_10": 0.04543, + "precision_at_100": 0.00746, + "precision_at_1000": 0.0011, + "precision_at_20": 0.02659, + "precision_at_3": 0.10945, + "precision_at_5": 0.07369, + "recall_at_1": 0.20827, + "recall_at_10": 0.37079, + "recall_at_100": 0.56696, + "recall_at_1000": 0.78233, + "recall_at_20": 0.42684, + "recall_at_3": 0.27246, + "recall_at_5": 0.30581 + } + ] + }, + "task_name": "CQADupstackUnixRetrieval" +} \ No newline at end of file diff --git a/results/bm25s/0_1_10/CQADupstackWebmastersRetrieval.json b/results/bm25s/0_1_10/CQADupstackWebmastersRetrieval.json new file mode 100644 index 0000000000..82a09c3be2 --- /dev/null +++ b/results/bm25s/0_1_10/CQADupstackWebmastersRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "160c094312a0e1facb97e55eeddb698c0abe3571", + "evaluation_time": 3.6046667098999023, + "kg_co2_emissions": null, + "mteb_version": "1.12.22", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.31381, + "map_at_1": 0.20401, + "map_at_10": 0.27149, + "map_at_100": 0.28247, + "map_at_1000": 0.28427, + "map_at_20": 0.27689, + "map_at_3": 0.25297, + "map_at_5": 0.26328, + "mrr_at_1": 0.2490118577075099, + "mrr_at_10": 0.3089121023903632, + "mrr_at_100": 0.3166700442443893, + "mrr_at_1000": 0.3175138863336789, + "mrr_at_20": 0.3129976879984529, + "mrr_at_3": 0.291501976284585, + "mrr_at_5": 0.3009881422924902, + "nauc_map_at_1000_diff1": 0.5279259696819044, + "nauc_map_at_1000_max": 0.46329475997406044, + "nauc_map_at_1000_std": 0.06547870908699323, + "nauc_map_at_100_diff1": 0.5283834923038861, + "nauc_map_at_100_max": 0.4641720431601167, + "nauc_map_at_100_std": 0.06607806394464566, + "nauc_map_at_10_diff1": 0.5353956046031676, + "nauc_map_at_10_max": 0.4630910435560766, + "nauc_map_at_10_std": 0.05899858152809982, + "nauc_map_at_1_diff1": 0.593718121175583, + "nauc_map_at_1_max": 0.45579268847285004, + "nauc_map_at_1_std": 0.026643765659804024, + "nauc_map_at_20_diff1": 0.5324300952469061, + "nauc_map_at_20_max": 0.4633369742886202, + "nauc_map_at_20_std": 0.06119339971514889, + "nauc_map_at_3_diff1": 0.5505536966653582, + "nauc_map_at_3_max": 0.4618139662527373, + "nauc_map_at_3_std": 0.03760893738372416, + "nauc_map_at_5_diff1": 0.5373025830714582, + "nauc_map_at_5_max": 0.465633874727416, + "nauc_map_at_5_std": 0.05693555077517676, + "nauc_mrr_at_1000_diff1": 0.5316356471559894, + "nauc_mrr_at_1000_max": 0.46451151429079285, + "nauc_mrr_at_1000_std": 0.07073417799809145, + "nauc_mrr_at_100_diff1": 0.5316581971698847, + "nauc_mrr_at_100_max": 0.4645936211111179, + "nauc_mrr_at_100_std": 0.0707440067513659, + "nauc_mrr_at_10_diff1": 0.5350221502915765, + "nauc_mrr_at_10_max": 0.46577083755681403, + "nauc_mrr_at_10_std": 0.06659626546294359, + "nauc_mrr_at_1_diff1": 0.5895269743691435, + "nauc_mrr_at_1_max": 0.48134372189240965, + "nauc_mrr_at_1_std": 0.057437057726498306, + "nauc_mrr_at_20_diff1": 0.5331272615996678, + "nauc_mrr_at_20_max": 0.46417654295230293, + "nauc_mrr_at_20_std": 0.06829318617730706, + "nauc_mrr_at_3_diff1": 0.5579090058076152, + "nauc_mrr_at_3_max": 0.4716105247172262, + "nauc_mrr_at_3_std": 0.05503112764040469, + "nauc_mrr_at_5_diff1": 0.5397509729119189, + "nauc_mrr_at_5_max": 0.4692567053849128, + "nauc_mrr_at_5_std": 0.06496455423854373, + "nauc_ndcg_at_1000_diff1": 0.48189117696631434, + "nauc_ndcg_at_1000_max": 0.4534814258381619, + "nauc_ndcg_at_1000_std": 0.10429747756362977, + "nauc_ndcg_at_100_diff1": 0.4872324744793197, + "nauc_ndcg_at_100_max": 0.4612716016816071, + "nauc_ndcg_at_100_std": 0.11163851701720763, + "nauc_ndcg_at_10_diff1": 0.5080882336575766, + "nauc_ndcg_at_10_max": 0.4604597653102985, + "nauc_ndcg_at_10_std": 0.0799899744444285, + "nauc_ndcg_at_1_diff1": 0.5895269743691435, + "nauc_ndcg_at_1_max": 0.48134372189240965, + "nauc_ndcg_at_1_std": 0.057437057726498306, + "nauc_ndcg_at_20_diff1": 0.49934075595552246, + "nauc_ndcg_at_20_max": 0.45648369057327015, + "nauc_ndcg_at_20_std": 0.0850420053948351, + "nauc_ndcg_at_3_diff1": 0.5428216142171366, + "nauc_ndcg_at_3_max": 0.47225578090252907, + "nauc_ndcg_at_3_std": 0.05010782296331936, + "nauc_ndcg_at_5_diff1": 0.518320080382509, + "nauc_ndcg_at_5_max": 0.4726037737370764, + "nauc_ndcg_at_5_std": 0.07923446996322413, + "nauc_precision_at_1000_diff1": -0.09228778704486294, + "nauc_precision_at_1000_max": -0.03508520167939041, + "nauc_precision_at_1000_std": 0.03956721148886218, + "nauc_precision_at_100_diff1": 0.06653976718002261, + "nauc_precision_at_100_max": 0.19550492011922999, + "nauc_precision_at_100_std": 0.17897968050386315, + "nauc_precision_at_10_diff1": 0.3198917297137462, + "nauc_precision_at_10_max": 0.3830276972249165, + "nauc_precision_at_10_std": 0.15021745959601646, + "nauc_precision_at_1_diff1": 0.5895269743691435, + "nauc_precision_at_1_max": 0.48134372189240965, + "nauc_precision_at_1_std": 0.057437057726498306, + "nauc_precision_at_20_diff1": 0.2165038898729356, + "nauc_precision_at_20_max": 0.31115030725591947, + "nauc_precision_at_20_std": 0.1536660681823508, + "nauc_precision_at_3_diff1": 0.46457083711144, + "nauc_precision_at_3_max": 0.4659599465284396, + "nauc_precision_at_3_std": 0.08949328588570968, + "nauc_precision_at_5_diff1": 0.38969953249077904, + "nauc_precision_at_5_max": 0.44824177975069496, + "nauc_precision_at_5_std": 0.15288322129514723, + "nauc_recall_at_1000_diff1": 0.14721295384521882, + "nauc_recall_at_1000_max": 0.3384860693955491, + "nauc_recall_at_1000_std": 0.32356731462853294, + "nauc_recall_at_100_diff1": 0.3163909260124831, + "nauc_recall_at_100_max": 0.4170589758776528, + "nauc_recall_at_100_std": 0.2725998909308775, + "nauc_recall_at_10_diff1": 0.41347511329133824, + "nauc_recall_at_10_max": 0.4202868086781561, + "nauc_recall_at_10_std": 0.12008876606122126, + "nauc_recall_at_1_diff1": 0.593718121175583, + "nauc_recall_at_1_max": 0.45579268847285004, + "nauc_recall_at_1_std": 0.026643765659804024, + "nauc_recall_at_20_diff1": 0.3865088759617077, + "nauc_recall_at_20_max": 0.40868574665327895, + "nauc_recall_at_20_std": 0.14244126049803962, + "nauc_recall_at_3_diff1": 0.5053670294344428, + "nauc_recall_at_3_max": 0.44894189844327304, + "nauc_recall_at_3_std": 0.046335483133441054, + "nauc_recall_at_5_diff1": 0.443776040501292, + "nauc_recall_at_5_max": 0.4546979176333201, + "nauc_recall_at_5_std": 0.1120983283983342, + "ndcg_at_1": 0.24901, + "ndcg_at_10": 0.31381, + "ndcg_at_100": 0.35919, + "ndcg_at_1000": 0.3903, + "ndcg_at_20": 0.32857, + "ndcg_at_3": 0.28334, + "ndcg_at_5": 0.29708, + "precision_at_1": 0.24901, + "precision_at_10": 0.05553, + "precision_at_100": 0.01051, + "precision_at_1000": 0.00191, + "precision_at_20": 0.03458, + "precision_at_3": 0.12912, + "precision_at_5": 0.0913, + "recall_at_1": 0.20401, + "recall_at_10": 0.39809, + "recall_at_100": 0.60315, + "recall_at_1000": 0.80514, + "recall_at_20": 0.45429, + "recall_at_3": 0.30235, + "recall_at_5": 0.34287 + } + ] + }, + "task_name": "CQADupstackWebmastersRetrieval" +} \ No newline at end of file diff --git a/results/bm25s/0_1_10/CQADupstackWordpressRetrieval.json b/results/bm25s/0_1_10/CQADupstackWordpressRetrieval.json new file mode 100644 index 0000000000..9f8315203a --- /dev/null +++ b/results/bm25s/0_1_10/CQADupstackWordpressRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "4ffe81d471b1924886b33c7567bfb200e9eec5c4", + "evaluation_time": 8.962841272354126, + "kg_co2_emissions": null, + "mteb_version": "1.12.22", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.27925, + "map_at_1": 0.18768, + "map_at_10": 0.24377, + "map_at_100": 0.25111, + "map_at_1000": 0.25213, + "map_at_20": 0.24751, + "map_at_3": 0.22457, + "map_at_5": 0.23537, + "mrr_at_1": 0.20517560073937152, + "mrr_at_10": 0.26217175718099933, + "mrr_at_100": 0.2692186004458705, + "mrr_at_1000": 0.2701321028478451, + "mrr_at_20": 0.26585398019532724, + "mrr_at_3": 0.2436845348120763, + "mrr_at_5": 0.2546826863832409, + "nauc_map_at_1000_diff1": 0.39655609517476276, + "nauc_map_at_1000_max": 0.3132242767390083, + "nauc_map_at_1000_std": 0.06490155837644618, + "nauc_map_at_100_diff1": 0.39663610169502517, + "nauc_map_at_100_max": 0.3129522916643771, + "nauc_map_at_100_std": 0.06440703486924076, + "nauc_map_at_10_diff1": 0.3985063940250327, + "nauc_map_at_10_max": 0.31236072423269473, + "nauc_map_at_10_std": 0.05900923323170141, + "nauc_map_at_1_diff1": 0.46187022090471624, + "nauc_map_at_1_max": 0.32601887008164754, + "nauc_map_at_1_std": 0.05385560578499423, + "nauc_map_at_20_diff1": 0.3977172766848172, + "nauc_map_at_20_max": 0.3141665911139833, + "nauc_map_at_20_std": 0.06265869233238652, + "nauc_map_at_3_diff1": 0.4224916529080315, + "nauc_map_at_3_max": 0.3133443955593098, + "nauc_map_at_3_std": 0.04960847336074426, + "nauc_map_at_5_diff1": 0.40480946271539603, + "nauc_map_at_5_max": 0.3107148561904914, + "nauc_map_at_5_std": 0.05285372946125663, + "nauc_mrr_at_1000_diff1": 0.4003759468531816, + "nauc_mrr_at_1000_max": 0.3116101043872568, + "nauc_mrr_at_1000_std": 0.06517246964626357, + "nauc_mrr_at_100_diff1": 0.4002097045121254, + "nauc_mrr_at_100_max": 0.3113034884634527, + "nauc_mrr_at_100_std": 0.06490846750485622, + "nauc_mrr_at_10_diff1": 0.40111728286636217, + "nauc_mrr_at_10_max": 0.3110651088143865, + "nauc_mrr_at_10_std": 0.06030961420087937, + "nauc_mrr_at_1_diff1": 0.4659211141404259, + "nauc_mrr_at_1_max": 0.31659568440682, + "nauc_mrr_at_1_std": 0.04526532568574501, + "nauc_mrr_at_20_diff1": 0.4002146309843532, + "nauc_mrr_at_20_max": 0.3123896122076063, + "nauc_mrr_at_20_std": 0.06363076643869292, + "nauc_mrr_at_3_diff1": 0.4221729804004896, + "nauc_mrr_at_3_max": 0.31601191436946774, + "nauc_mrr_at_3_std": 0.05498660098196729, + "nauc_mrr_at_5_diff1": 0.4059915316000488, + "nauc_mrr_at_5_max": 0.31336615560268666, + "nauc_mrr_at_5_std": 0.05716760564963853, + "nauc_ndcg_at_1000_diff1": 0.3640361331187997, + "nauc_ndcg_at_1000_max": 0.318977809686213, + "nauc_ndcg_at_1000_std": 0.10668299187107794, + "nauc_ndcg_at_100_diff1": 0.3623936219944013, + "nauc_ndcg_at_100_max": 0.30849905286213425, + "nauc_ndcg_at_100_std": 0.09392555029122387, + "nauc_ndcg_at_10_diff1": 0.3674877076899867, + "nauc_ndcg_at_10_max": 0.30691445346058965, + "nauc_ndcg_at_10_std": 0.06771751502527852, + "nauc_ndcg_at_1_diff1": 0.4659211141404259, + "nauc_ndcg_at_1_max": 0.31659568440682, + "nauc_ndcg_at_1_std": 0.04526532568574501, + "nauc_ndcg_at_20_diff1": 0.36673383728363496, + "nauc_ndcg_at_20_max": 0.3133166626168057, + "nauc_ndcg_at_20_std": 0.07948815825889607, + "nauc_ndcg_at_3_diff1": 0.4095639421483382, + "nauc_ndcg_at_3_max": 0.3117284897194662, + "nauc_ndcg_at_3_std": 0.05198577276832792, + "nauc_ndcg_at_5_diff1": 0.3786237054952364, + "nauc_ndcg_at_5_max": 0.30549670383889327, + "nauc_ndcg_at_5_std": 0.055353206060983595, + "nauc_precision_at_1000_diff1": 0.016201240949539633, + "nauc_precision_at_1000_max": 0.14651004751471747, + "nauc_precision_at_1000_std": 0.21232019794105428, + "nauc_precision_at_100_diff1": 0.1463316388097302, + "nauc_precision_at_100_max": 0.21219033642556587, + "nauc_precision_at_100_std": 0.17572027527417455, + "nauc_precision_at_10_diff1": 0.23903309787022933, + "nauc_precision_at_10_max": 0.27153174252660917, + "nauc_precision_at_10_std": 0.10005907404948872, + "nauc_precision_at_1_diff1": 0.4659211141404259, + "nauc_precision_at_1_max": 0.31659568440682, + "nauc_precision_at_1_std": 0.04526532568574501, + "nauc_precision_at_20_diff1": 0.21964004660623634, + "nauc_precision_at_20_max": 0.2840489708496851, + "nauc_precision_at_20_std": 0.13458227993298227, + "nauc_precision_at_3_diff1": 0.36160586200333944, + "nauc_precision_at_3_max": 0.2839553331541429, + "nauc_precision_at_3_std": 0.04517171322873024, + "nauc_precision_at_5_diff1": 0.28657612642183605, + "nauc_precision_at_5_max": 0.27732315860076506, + "nauc_precision_at_5_std": 0.06327093806110082, + "nauc_recall_at_1000_diff1": 0.20799069061471737, + "nauc_recall_at_1000_max": 0.4210585952086403, + "nauc_recall_at_1000_std": 0.4421280238210987, + "nauc_recall_at_100_diff1": 0.2550001546225366, + "nauc_recall_at_100_max": 0.2927263272882623, + "nauc_recall_at_100_std": 0.2086822058660352, + "nauc_recall_at_10_diff1": 0.2773290418775887, + "nauc_recall_at_10_max": 0.2849451658353981, + "nauc_recall_at_10_std": 0.08771470269125098, + "nauc_recall_at_1_diff1": 0.46187022090471624, + "nauc_recall_at_1_max": 0.32601887008164754, + "nauc_recall_at_1_std": 0.05385560578499423, + "nauc_recall_at_20_diff1": 0.2789717932623959, + "nauc_recall_at_20_max": 0.3072205169567785, + "nauc_recall_at_20_std": 0.12694770547061668, + "nauc_recall_at_3_diff1": 0.3708068123997188, + "nauc_recall_at_3_max": 0.2999598017363359, + "nauc_recall_at_3_std": 0.05089611122918276, + "nauc_recall_at_5_diff1": 0.30010646741033453, + "nauc_recall_at_5_max": 0.2788680456880977, + "nauc_recall_at_5_std": 0.05398748309586544, + "ndcg_at_1": 0.20518, + "ndcg_at_10": 0.27925, + "ndcg_at_100": 0.31784, + "ndcg_at_1000": 0.34844, + "ndcg_at_20": 0.29229, + "ndcg_at_3": 0.24074, + "ndcg_at_5": 0.25968, + "precision_at_1": 0.20518, + "precision_at_10": 0.04251, + "precision_at_100": 0.0066, + "precision_at_1000": 0.00098, + "precision_at_20": 0.0244, + "precision_at_3": 0.09982, + "precision_at_5": 0.07024, + "recall_at_1": 0.18768, + "recall_at_10": 0.3719, + "recall_at_100": 0.55194, + "recall_at_1000": 0.7908, + "recall_at_20": 0.42079, + "recall_at_3": 0.26784, + "recall_at_5": 0.3138 + } + ] + }, + "task_name": "CQADupstackWordpressRetrieval" +} \ No newline at end of file diff --git a/results/bm25s/0_1_10/ClimateFEVER.json b/results/bm25s/0_1_10/ClimateFEVER.json new file mode 100644 index 0000000000..3b4288d094 --- /dev/null +++ b/results/bm25s/0_1_10/ClimateFEVER.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "47f2ac6acb640fc46020b02a5b59fdda04d39380", + "evaluation_time": 658.6776967048645, + "kg_co2_emissions": null, + "mteb_version": "1.12.22", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.13621, + "map_at_1": 0.05313, + "map_at_10": 0.09163, + "map_at_100": 0.10205, + "map_at_1000": 0.10362, + "map_at_20": 0.09667, + "map_at_3": 0.07708, + "map_at_5": 0.08378, + "mrr_at_1": 0.11465798045602606, + "mrr_at_10": 0.18459826275787178, + "mrr_at_100": 0.19568921572969208, + "mrr_at_1000": 0.19658156348118172, + "mrr_at_20": 0.19080215613895668, + "mrr_at_3": 0.1606948968512488, + "mrr_at_5": 0.17388707926167207, + "nauc_map_at_1000_diff1": 0.247226426004553, + "nauc_map_at_1000_max": 0.27868999301771036, + "nauc_map_at_1000_std": 0.18654241852502532, + "nauc_map_at_100_diff1": 0.24693146913651848, + "nauc_map_at_100_max": 0.277249566216726, + "nauc_map_at_100_std": 0.1844048293164565, + "nauc_map_at_10_diff1": 0.24879459553381905, + "nauc_map_at_10_max": 0.26832269006996207, + "nauc_map_at_10_std": 0.16596082380735555, + "nauc_map_at_1_diff1": 0.3440887809568764, + "nauc_map_at_1_max": 0.28765127279282576, + "nauc_map_at_1_std": 0.16328214802465718, + "nauc_map_at_20_diff1": 0.24711393091192757, + "nauc_map_at_20_max": 0.2720331515741425, + "nauc_map_at_20_std": 0.17282896482415844, + "nauc_map_at_3_diff1": 0.2645376815930281, + "nauc_map_at_3_max": 0.2647503125354659, + "nauc_map_at_3_std": 0.15106824771902935, + "nauc_map_at_5_diff1": 0.2544041542650692, + "nauc_map_at_5_max": 0.2696340749081419, + "nauc_map_at_5_std": 0.15872080576103006, + "nauc_mrr_at_1000_diff1": 0.2619535709011585, + "nauc_mrr_at_1000_max": 0.28646889408113346, + "nauc_mrr_at_1000_std": 0.20242450990154975, + "nauc_mrr_at_100_diff1": 0.2618140777939259, + "nauc_mrr_at_100_max": 0.2863585743925081, + "nauc_mrr_at_100_std": 0.20246444713897058, + "nauc_mrr_at_10_diff1": 0.2602671871768324, + "nauc_mrr_at_10_max": 0.28525483906474225, + "nauc_mrr_at_10_std": 0.19621516267233807, + "nauc_mrr_at_1_diff1": 0.3391788289712938, + "nauc_mrr_at_1_max": 0.3011450966089375, + "nauc_mrr_at_1_std": 0.1858556459127588, + "nauc_mrr_at_20_diff1": 0.2612871146699057, + "nauc_mrr_at_20_max": 0.2850650380983841, + "nauc_mrr_at_20_std": 0.19926847069239884, + "nauc_mrr_at_3_diff1": 0.27394259799250914, + "nauc_mrr_at_3_max": 0.2808670271867963, + "nauc_mrr_at_3_std": 0.18331819893247742, + "nauc_mrr_at_5_diff1": 0.26565967987490036, + "nauc_mrr_at_5_max": 0.2883697263379934, + "nauc_mrr_at_5_std": 0.19259092559874788, + "nauc_ndcg_at_1000_diff1": 0.22611371741551745, + "nauc_ndcg_at_1000_max": 0.30537379855758595, + "nauc_ndcg_at_1000_std": 0.2653000124131748, + "nauc_ndcg_at_100_diff1": 0.2203227798470496, + "nauc_ndcg_at_100_max": 0.28733018305668134, + "nauc_ndcg_at_100_std": 0.24167187283751904, + "nauc_ndcg_at_10_diff1": 0.22527025509159324, + "nauc_ndcg_at_10_max": 0.26650659009666994, + "nauc_ndcg_at_10_std": 0.1823997115126037, + "nauc_ndcg_at_1_diff1": 0.3391788289712938, + "nauc_ndcg_at_1_max": 0.3011450966089375, + "nauc_ndcg_at_1_std": 0.1858556459127588, + "nauc_ndcg_at_20_diff1": 0.222035480376174, + "nauc_ndcg_at_20_max": 0.27116090721792985, + "nauc_ndcg_at_20_std": 0.1965003923924903, + "nauc_ndcg_at_3_diff1": 0.2552028770739362, + "nauc_ndcg_at_3_max": 0.2679874643218914, + "nauc_ndcg_at_3_std": 0.16380666394718033, + "nauc_ndcg_at_5_diff1": 0.23679727408237958, + "nauc_ndcg_at_5_max": 0.27143234340293676, + "nauc_ndcg_at_5_std": 0.1705653747848498, + "nauc_precision_at_1000_diff1": 0.10449293428539591, + "nauc_precision_at_1000_max": 0.29274142567391365, + "nauc_precision_at_1000_std": 0.37005548819288075, + "nauc_precision_at_100_diff1": 0.13483613194808983, + "nauc_precision_at_100_max": 0.3036400950287313, + "nauc_precision_at_100_std": 0.3590951228503246, + "nauc_precision_at_10_diff1": 0.1655368802120059, + "nauc_precision_at_10_max": 0.28583926840169815, + "nauc_precision_at_10_std": 0.24497627587670562, + "nauc_precision_at_1_diff1": 0.3391788289712938, + "nauc_precision_at_1_max": 0.3011450966089375, + "nauc_precision_at_1_std": 0.1858556459127588, + "nauc_precision_at_20_diff1": 0.15363704566104291, + "nauc_precision_at_20_max": 0.2853401443490719, + "nauc_precision_at_20_std": 0.26785064306143563, + "nauc_precision_at_3_diff1": 0.21837846425081575, + "nauc_precision_at_3_max": 0.2878307082990253, + "nauc_precision_at_3_std": 0.19590804609446316, + "nauc_precision_at_5_diff1": 0.19438468252050528, + "nauc_precision_at_5_max": 0.31023872971951844, + "nauc_precision_at_5_std": 0.2275989118453553, + "nauc_recall_at_1000_diff1": 0.1282535104317408, + "nauc_recall_at_1000_max": 0.26383903793527697, + "nauc_recall_at_1000_std": 0.3350016837668618, + "nauc_recall_at_100_diff1": 0.12078929128761444, + "nauc_recall_at_100_max": 0.22224414527368924, + "nauc_recall_at_100_std": 0.2696166211100714, + "nauc_recall_at_10_diff1": 0.15392169882993384, + "nauc_recall_at_10_max": 0.215246051931427, + "nauc_recall_at_10_std": 0.16063491645798894, + "nauc_recall_at_1_diff1": 0.3440887809568764, + "nauc_recall_at_1_max": 0.28765127279282576, + "nauc_recall_at_1_std": 0.16328214802465718, + "nauc_recall_at_20_diff1": 0.1429410231490379, + "nauc_recall_at_20_max": 0.2161149454961278, + "nauc_recall_at_20_std": 0.18122298364484674, + "nauc_recall_at_3_diff1": 0.20878449064242663, + "nauc_recall_at_3_max": 0.2269393818095996, + "nauc_recall_at_3_std": 0.12885540473047072, + "nauc_recall_at_5_diff1": 0.17860744664170547, + "nauc_recall_at_5_max": 0.232243869670833, + "nauc_recall_at_5_std": 0.14278866305910592, + "ndcg_at_1": 0.11466, + "ndcg_at_10": 0.13621, + "ndcg_at_100": 0.18956, + "ndcg_at_1000": 0.22607, + "ndcg_at_20": 0.15387, + "ndcg_at_3": 0.10713, + "ndcg_at_5": 0.11664, + "precision_at_1": 0.11466, + "precision_at_10": 0.04293, + "precision_at_100": 0.00988, + "precision_at_1000": 0.00165, + "precision_at_20": 0.02896, + "precision_at_3": 0.07796, + "precision_at_5": 0.06124, + "recall_at_1": 0.05313, + "recall_at_10": 0.17342, + "recall_at_100": 0.36655, + "recall_at_1000": 0.57759, + "recall_at_20": 0.22326, + "recall_at_3": 0.1027, + "recall_at_5": 0.12795 + } + ] + }, + "task_name": "ClimateFEVER" +} \ No newline at end of file diff --git a/results/bm25s/0_1_10/DBPedia.json b/results/bm25s/0_1_10/DBPedia.json new file mode 100644 index 0000000000..c9278b7e83 --- /dev/null +++ b/results/bm25s/0_1_10/DBPedia.json @@ -0,0 +1,307 @@ +{ + "dataset_revision": "c0f706b76e590d620bd6618b3ca8efdd34e2d659", + "evaluation_time": 715.9223916530609, + "kg_co2_emissions": null, + "mteb_version": "1.12.22", + "scores": { + "dev": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.34447, + "map_at_1": 0.07331, + "map_at_10": 0.15437, + "map_at_100": 0.2375, + "map_at_1000": 0.25237, + "map_at_20": 0.18247, + "map_at_3": 0.10689, + "map_at_5": 0.12419, + "mrr_at_1": 0.47761194029850745, + "mrr_at_10": 0.6259950248756221, + "mrr_at_100": 0.6297157630899706, + "mrr_at_1000": 0.6297157630899706, + "mrr_at_20": 0.6286749451124944, + "mrr_at_3": 0.5970149253731343, + "mrr_at_5": 0.614179104477612, + "nauc_map_at_1000_diff1": 0.2308805166876326, + "nauc_map_at_1000_max": 0.10197415146319086, + "nauc_map_at_1000_std": -0.008803683376396977, + "nauc_map_at_100_diff1": 0.24363820813822898, + "nauc_map_at_100_max": 0.09234743358758254, + "nauc_map_at_100_std": -0.044784639460081196, + "nauc_map_at_10_diff1": 0.34233304413759424, + "nauc_map_at_10_max": 0.1435550500971572, + "nauc_map_at_10_std": -0.1447974788196049, + "nauc_map_at_1_diff1": 0.4089870534392689, + "nauc_map_at_1_max": 0.1616824380442796, + "nauc_map_at_1_std": -0.23213204365999618, + "nauc_map_at_20_diff1": 0.315528489485945, + "nauc_map_at_20_max": 0.1371219637700406, + "nauc_map_at_20_std": -0.12013967107149916, + "nauc_map_at_3_diff1": 0.37984118755244567, + "nauc_map_at_3_max": 0.16447657439642535, + "nauc_map_at_3_std": -0.19024167802604572, + "nauc_map_at_5_diff1": 0.3590418608454131, + "nauc_map_at_5_max": 0.14885206181889454, + "nauc_map_at_5_std": -0.15644479745737988, + "nauc_mrr_at_1000_diff1": 0.3447295853894599, + "nauc_mrr_at_1000_max": 0.24776728438275256, + "nauc_mrr_at_1000_std": 0.09134046572067292, + "nauc_mrr_at_100_diff1": 0.3447295853894599, + "nauc_mrr_at_100_max": 0.24776728438275256, + "nauc_mrr_at_100_std": 0.09134046572067292, + "nauc_mrr_at_10_diff1": 0.3491156427676268, + "nauc_mrr_at_10_max": 0.25110534386844524, + "nauc_mrr_at_10_std": 0.09795514108963206, + "nauc_mrr_at_1_diff1": 0.2537145004004366, + "nauc_mrr_at_1_max": 0.22316714600199256, + "nauc_mrr_at_1_std": 0.06757506903838825, + "nauc_mrr_at_20_diff1": 0.344451443387572, + "nauc_mrr_at_20_max": 0.2488698626169694, + "nauc_mrr_at_20_std": 0.09220764359500463, + "nauc_mrr_at_3_diff1": 0.3588278429182936, + "nauc_mrr_at_3_max": 0.20070641898190145, + "nauc_mrr_at_3_std": 0.04700500938532884, + "nauc_mrr_at_5_diff1": 0.3529950651596765, + "nauc_mrr_at_5_max": 0.23171006107015915, + "nauc_mrr_at_5_std": 0.07870095650408343, + "nauc_ndcg_at_1000_diff1": 0.19907606801469166, + "nauc_ndcg_at_1000_max": 0.143023133118224, + "nauc_ndcg_at_1000_std": 0.15006187081229308, + "nauc_ndcg_at_100_diff1": 0.2378879133133047, + "nauc_ndcg_at_100_max": 0.08681338570550308, + "nauc_ndcg_at_100_std": 0.06713995742799864, + "nauc_ndcg_at_10_diff1": 0.29067414152935506, + "nauc_ndcg_at_10_max": 0.13415857622474373, + "nauc_ndcg_at_10_std": 0.04804177882954882, + "nauc_ndcg_at_1_diff1": 0.2890135303742012, + "nauc_ndcg_at_1_max": 0.21228647821183833, + "nauc_ndcg_at_1_std": 0.06673980542934974, + "nauc_ndcg_at_20_diff1": 0.27886690449781104, + "nauc_ndcg_at_20_max": 0.12458272831499184, + "nauc_ndcg_at_20_std": 0.0422380646851309, + "nauc_ndcg_at_3_diff1": 0.2727542469630014, + "nauc_ndcg_at_3_max": 0.16346695203126302, + "nauc_ndcg_at_3_std": 0.05691938264106389, + "nauc_ndcg_at_5_diff1": 0.26995928259995466, + "nauc_ndcg_at_5_max": 0.15312546493670662, + "nauc_ndcg_at_5_std": 0.05922649676329816, + "nauc_precision_at_1000_diff1": -0.06332506282924556, + "nauc_precision_at_1000_max": 0.14597119789401242, + "nauc_precision_at_1000_std": 0.29498038081664774, + "nauc_precision_at_100_diff1": -0.03888064280813274, + "nauc_precision_at_100_max": 0.03937738580114327, + "nauc_precision_at_100_std": 0.26485917785339114, + "nauc_precision_at_10_diff1": 0.016833445252033834, + "nauc_precision_at_10_max": 0.06696064629930217, + "nauc_precision_at_10_std": 0.13590963374049392, + "nauc_precision_at_1_diff1": 0.2537145004004366, + "nauc_precision_at_1_max": 0.22316714600199256, + "nauc_precision_at_1_std": 0.06757506903838825, + "nauc_precision_at_20_diff1": 0.0015590779062594968, + "nauc_precision_at_20_max": 0.08543738737066613, + "nauc_precision_at_20_std": 0.1917296176189857, + "nauc_precision_at_3_diff1": 0.03769401066043022, + "nauc_precision_at_3_max": 0.05686575926010372, + "nauc_precision_at_3_std": 0.10909383111935025, + "nauc_precision_at_5_diff1": 0.03847474391378747, + "nauc_precision_at_5_max": 0.061220319774352217, + "nauc_precision_at_5_std": 0.064935955489422, + "nauc_recall_at_1000_diff1": -0.09252772137224734, + "nauc_recall_at_1000_max": -0.06170258460534573, + "nauc_recall_at_1000_std": 0.3029515546122396, + "nauc_recall_at_100_diff1": 0.0406042019831963, + "nauc_recall_at_100_max": -0.11114658109286552, + "nauc_recall_at_100_std": 0.1177416235951467, + "nauc_recall_at_10_diff1": 0.20681006870599097, + "nauc_recall_at_10_max": 0.01780482552786911, + "nauc_recall_at_10_std": -0.03079836470244515, + "nauc_recall_at_1_diff1": 0.4089870534392689, + "nauc_recall_at_1_max": 0.1616824380442796, + "nauc_recall_at_1_std": -0.23213204365999618, + "nauc_recall_at_20_diff1": 0.14283501976132232, + "nauc_recall_at_20_max": 0.009696755268985788, + "nauc_recall_at_20_std": -0.0016450805848480885, + "nauc_recall_at_3_diff1": 0.3706060817732486, + "nauc_recall_at_3_max": 0.14970695432559245, + "nauc_recall_at_3_std": -0.18041991606129326, + "nauc_recall_at_5_diff1": 0.28424409870427214, + "nauc_recall_at_5_max": 0.06621544737321075, + "nauc_recall_at_5_std": -0.004969375162336681, + "ndcg_at_1": 0.3806, + "ndcg_at_10": 0.34447, + "ndcg_at_100": 0.4303, + "ndcg_at_1000": 0.49764, + "ndcg_at_20": 0.35433, + "ndcg_at_3": 0.35427, + "ndcg_at_5": 0.34033, + "precision_at_1": 0.47761, + "precision_at_10": 0.28209, + "precision_at_100": 0.09955, + "precision_at_1000": 0.01572, + "precision_at_20": 0.22836, + "precision_at_3": 0.39303, + "precision_at_5": 0.33134, + "recall_at_1": 0.07331, + "recall_at_10": 0.24317, + "recall_at_100": 0.55033, + "recall_at_1000": 0.76351, + "recall_at_20": 0.3218, + "recall_at_3": 0.12679, + "recall_at_5": 0.16476 + } + ], + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.29907, + "map_at_1": 0.05855, + "map_at_10": 0.136, + "map_at_100": 0.20748, + "map_at_1000": 0.21921, + "map_at_20": 0.16481, + "map_at_3": 0.09244, + "map_at_5": 0.10913, + "mrr_at_1": 0.47, + "mrr_at_10": 0.5753779761904763, + "mrr_at_100": 0.5813791208348534, + "mrr_at_1000": 0.5815820216943877, + "mrr_at_20": 0.5789606025531916, + "mrr_at_3": 0.5500000000000002, + "mrr_at_5": 0.5653750000000002, + "nauc_map_at_1000_diff1": 0.2077520590773007, + "nauc_map_at_1000_max": 0.2627613158183316, + "nauc_map_at_1000_std": 0.16819577768276373, + "nauc_map_at_100_diff1": 0.21951107046390758, + "nauc_map_at_100_max": 0.25372359251382054, + "nauc_map_at_100_std": 0.14369641258870364, + "nauc_map_at_10_diff1": 0.33978282955960465, + "nauc_map_at_10_max": 0.23193065437420915, + "nauc_map_at_10_std": 0.006247325492942982, + "nauc_map_at_1_diff1": 0.48974139878730844, + "nauc_map_at_1_max": 0.22856688625561367, + "nauc_map_at_1_std": -0.08497687328629445, + "nauc_map_at_20_diff1": 0.28368049255617106, + "nauc_map_at_20_max": 0.2288152088860196, + "nauc_map_at_20_std": 0.04416267668220651, + "nauc_map_at_3_diff1": 0.432418616375195, + "nauc_map_at_3_max": 0.22310970445527872, + "nauc_map_at_3_std": -0.07304417209740263, + "nauc_map_at_5_diff1": 0.4011768608027594, + "nauc_map_at_5_max": 0.2302322589235516, + "nauc_map_at_5_std": -0.036551706832935286, + "nauc_mrr_at_1000_diff1": 0.23856933207366737, + "nauc_mrr_at_1000_max": 0.34649065214850394, + "nauc_mrr_at_1000_std": 0.20219081795726698, + "nauc_mrr_at_100_diff1": 0.23854901878689203, + "nauc_mrr_at_100_max": 0.3464934976404373, + "nauc_mrr_at_100_std": 0.2021788025647238, + "nauc_mrr_at_10_diff1": 0.23886606379073566, + "nauc_mrr_at_10_max": 0.34775977236670497, + "nauc_mrr_at_10_std": 0.201092127961341, + "nauc_mrr_at_1_diff1": 0.27937802449926813, + "nauc_mrr_at_1_max": 0.36258059837286893, + "nauc_mrr_at_1_std": 0.22787088437347525, + "nauc_mrr_at_20_diff1": 0.23897131436834726, + "nauc_mrr_at_20_max": 0.3464968365948819, + "nauc_mrr_at_20_std": 0.2007865497290927, + "nauc_mrr_at_3_diff1": 0.2414790908483059, + "nauc_mrr_at_3_max": 0.3339565960779817, + "nauc_mrr_at_3_std": 0.1896348539989109, + "nauc_mrr_at_5_diff1": 0.24044730658637503, + "nauc_mrr_at_5_max": 0.3460879001601124, + "nauc_mrr_at_5_std": 0.19649655112961686, + "nauc_ndcg_at_1000_diff1": 0.18580536819295784, + "nauc_ndcg_at_1000_max": 0.3063790189091747, + "nauc_ndcg_at_1000_std": 0.2601519419159607, + "nauc_ndcg_at_100_diff1": 0.2210083024468675, + "nauc_ndcg_at_100_max": 0.30148660499134833, + "nauc_ndcg_at_100_std": 0.20934235167883913, + "nauc_ndcg_at_10_diff1": 0.21821105559003678, + "nauc_ndcg_at_10_max": 0.2627080148405724, + "nauc_ndcg_at_10_std": 0.12061400186839254, + "nauc_ndcg_at_1_diff1": 0.31157516492837106, + "nauc_ndcg_at_1_max": 0.29515711528004807, + "nauc_ndcg_at_1_std": 0.13371999922572972, + "nauc_ndcg_at_20_diff1": 0.23513414238271238, + "nauc_ndcg_at_20_max": 0.2641463425203306, + "nauc_ndcg_at_20_std": 0.12029695023031989, + "nauc_ndcg_at_3_diff1": 0.24114647080505028, + "nauc_ndcg_at_3_max": 0.26672752915878273, + "nauc_ndcg_at_3_std": 0.11894402944520167, + "nauc_ndcg_at_5_diff1": 0.2232000934680274, + "nauc_ndcg_at_5_max": 0.2640131339017929, + "nauc_ndcg_at_5_std": 0.12590755801847703, + "nauc_precision_at_1000_diff1": -0.1900996802547713, + "nauc_precision_at_1000_max": 0.08191838599097585, + "nauc_precision_at_1000_std": 0.2484420424420171, + "nauc_precision_at_100_diff1": -0.10359196552923504, + "nauc_precision_at_100_max": 0.20526237090233418, + "nauc_precision_at_100_std": 0.3572944964921371, + "nauc_precision_at_10_diff1": -0.05490366769398425, + "nauc_precision_at_10_max": 0.21020279589300014, + "nauc_precision_at_10_std": 0.26412739021607723, + "nauc_precision_at_1_diff1": 0.2861909169057274, + "nauc_precision_at_1_max": 0.34684588250405934, + "nauc_precision_at_1_std": 0.21317506480160994, + "nauc_precision_at_20_diff1": -0.06820274659651028, + "nauc_precision_at_20_max": 0.19442190768895734, + "nauc_precision_at_20_std": 0.2757273814773303, + "nauc_precision_at_3_diff1": 0.07321664103901979, + "nauc_precision_at_3_max": 0.2642184997930874, + "nauc_precision_at_3_std": 0.21190349980000747, + "nauc_precision_at_5_diff1": 0.004382901462117628, + "nauc_precision_at_5_max": 0.23329997976405095, + "nauc_precision_at_5_std": 0.24511236547056983, + "nauc_recall_at_1000_diff1": 0.09775654517094699, + "nauc_recall_at_1000_max": 0.2547816807902218, + "nauc_recall_at_1000_std": 0.36663170297395486, + "nauc_recall_at_100_diff1": 0.15920389017527634, + "nauc_recall_at_100_max": 0.2575503522504537, + "nauc_recall_at_100_std": 0.23647893592039965, + "nauc_recall_at_10_diff1": 0.29076795175992065, + "nauc_recall_at_10_max": 0.21642084308176215, + "nauc_recall_at_10_std": 0.015898713043741908, + "nauc_recall_at_1_diff1": 0.48974139878730844, + "nauc_recall_at_1_max": 0.22856688625561367, + "nauc_recall_at_1_std": -0.08497687328629445, + "nauc_recall_at_20_diff1": 0.2471656989010021, + "nauc_recall_at_20_max": 0.20084823957811906, + "nauc_recall_at_20_std": 0.035575685285934217, + "nauc_recall_at_3_diff1": 0.3849242784573741, + "nauc_recall_at_3_max": 0.19492342641501698, + "nauc_recall_at_3_std": -0.08328256073488893, + "nauc_recall_at_5_diff1": 0.3516712573841502, + "nauc_recall_at_5_max": 0.20188615231788712, + "nauc_recall_at_5_std": -0.04199453303729504, + "ndcg_at_1": 0.35125, + "ndcg_at_10": 0.29907, + "ndcg_at_100": 0.35478, + "ndcg_at_1000": 0.41766, + "ndcg_at_20": 0.30465, + "ndcg_at_3": 0.31319, + "ndcg_at_5": 0.30124, + "precision_at_1": 0.4675, + "precision_at_10": 0.2645, + "precision_at_100": 0.08983, + "precision_at_1000": 0.01568, + "precision_at_20": 0.21087, + "precision_at_3": 0.36583, + "precision_at_5": 0.3205, + "recall_at_1": 0.05855, + "recall_at_10": 0.19087, + "recall_at_100": 0.44672, + "recall_at_1000": 0.66336, + "recall_at_20": 0.2649, + "recall_at_3": 0.10583, + "recall_at_5": 0.13506 + } + ] + }, + "task_name": "DBPedia" +} \ No newline at end of file diff --git a/results/bm25s/0_1_10/FEVER.json b/results/bm25s/0_1_10/FEVER.json new file mode 100644 index 0000000000..feac0a6d06 --- /dev/null +++ b/results/bm25s/0_1_10/FEVER.json @@ -0,0 +1,456 @@ +{ + "dataset_revision": "bea83ef9e8fb933d90a2f1d5515737465d613e12", + "evaluation_time": 5407.60852766037, + "kg_co2_emissions": null, + "mteb_version": "1.12.22", + "scores": { + "dev": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.49778, + "map_at_1": 0.33423, + "map_at_10": 0.43791, + "map_at_100": 0.44725, + "map_at_1000": 0.44775, + "map_at_20": 0.4433, + "map_at_3": 0.40968, + "map_at_5": 0.42574, + "mrr_at_1": 0.35838583858385836, + "mrr_at_10": 0.46525468022992883, + "mrr_at_100": 0.474008121462713, + "mrr_at_1000": 0.4743767003938272, + "mrr_at_20": 0.47038829232999, + "mrr_at_3": 0.43691869186918864, + "mrr_at_5": 0.4531728172817291, + "nauc_map_at_1000_diff1": 0.5347329553851071, + "nauc_map_at_1000_max": 0.33784431082359184, + "nauc_map_at_1000_std": -0.012185933362794553, + "nauc_map_at_100_diff1": 0.5346891930946847, + "nauc_map_at_100_max": 0.33785943915055844, + "nauc_map_at_100_std": -0.012317038593889804, + "nauc_map_at_10_diff1": 0.5335331416580519, + "nauc_map_at_10_max": 0.3360942856029575, + "nauc_map_at_10_std": -0.01593809871477947, + "nauc_map_at_1_diff1": 0.559164843726652, + "nauc_map_at_1_max": 0.3139070699510342, + "nauc_map_at_1_std": -0.014207864004640072, + "nauc_map_at_20_diff1": 0.5344178665644932, + "nauc_map_at_20_max": 0.33755114862971, + "nauc_map_at_20_std": -0.013745765518608578, + "nauc_map_at_3_diff1": 0.5331262749315363, + "nauc_map_at_3_max": 0.32651630045722974, + "nauc_map_at_3_std": -0.024230046099893737, + "nauc_map_at_5_diff1": 0.5325321298088999, + "nauc_map_at_5_max": 0.3326454636256423, + "nauc_map_at_5_std": -0.019204330563924094, + "nauc_mrr_at_1000_diff1": 0.5459938428436988, + "nauc_mrr_at_1000_max": 0.344481425333191, + "nauc_mrr_at_1000_std": -0.016023704745701525, + "nauc_mrr_at_100_diff1": 0.5459821277217941, + "nauc_mrr_at_100_max": 0.3445721517058422, + "nauc_mrr_at_100_std": -0.01602462754795248, + "nauc_mrr_at_10_diff1": 0.5444916628245736, + "nauc_mrr_at_10_max": 0.34361788215849365, + "nauc_mrr_at_10_std": -0.018323135062009045, + "nauc_mrr_at_1_diff1": 0.5745307789711923, + "nauc_mrr_at_1_max": 0.32468856331756, + "nauc_mrr_at_1_std": -0.015215958632884153, + "nauc_mrr_at_20_diff1": 0.5455470393896237, + "nauc_mrr_at_20_max": 0.3444688358073275, + "nauc_mrr_at_20_std": -0.017075718220351546, + "nauc_mrr_at_3_diff1": 0.5452616321099959, + "nauc_mrr_at_3_max": 0.3371531897141649, + "nauc_mrr_at_3_std": -0.023965156390319275, + "nauc_mrr_at_5_diff1": 0.5434552533862159, + "nauc_mrr_at_5_max": 0.3407757348748981, + "nauc_mrr_at_5_std": -0.02083263441435125, + "nauc_ndcg_at_1000_diff1": 0.5340328861725582, + "nauc_ndcg_at_1000_max": 0.3567385128102016, + "nauc_ndcg_at_1000_std": 0.007988692923404851, + "nauc_ndcg_at_100_diff1": 0.5336189176042889, + "nauc_ndcg_at_100_max": 0.3593537694147321, + "nauc_ndcg_at_100_std": 0.008214646240406794, + "nauc_ndcg_at_10_diff1": 0.5276283717978716, + "nauc_ndcg_at_10_max": 0.35201268849663725, + "nauc_ndcg_at_10_std": -0.009502801646387022, + "nauc_ndcg_at_1_diff1": 0.5745307789711923, + "nauc_ndcg_at_1_max": 0.32468856331756, + "nauc_ndcg_at_1_std": -0.015215958632884153, + "nauc_ndcg_at_20_diff1": 0.5308142309723979, + "nauc_ndcg_at_20_max": 0.3574929713838812, + "nauc_ndcg_at_20_std": -0.0018355073549611732, + "nauc_ndcg_at_3_diff1": 0.5268665334964149, + "nauc_ndcg_at_3_max": 0.3328461858197005, + "nauc_ndcg_at_3_std": -0.02620105968996771, + "nauc_ndcg_at_5_diff1": 0.5250962480573081, + "nauc_ndcg_at_5_max": 0.3429813491402346, + "nauc_ndcg_at_5_std": -0.018147037144491156, + "nauc_precision_at_1000_diff1": -0.03219929187170476, + "nauc_precision_at_1000_max": 0.060982344142513176, + "nauc_precision_at_1000_std": 0.146608919777841, + "nauc_precision_at_100_diff1": 0.20657084539876217, + "nauc_precision_at_100_max": 0.26730979640101243, + "nauc_precision_at_100_std": 0.16115178344732714, + "nauc_precision_at_10_diff1": 0.44725460596620065, + "nauc_precision_at_10_max": 0.38332584575839773, + "nauc_precision_at_10_std": 0.0213281214415001, + "nauc_precision_at_1_diff1": 0.5745307789711923, + "nauc_precision_at_1_max": 0.32468856331756, + "nauc_precision_at_1_std": -0.015215958632884153, + "nauc_precision_at_20_diff1": 0.39622758547322784, + "nauc_precision_at_20_max": 0.3793911229399525, + "nauc_precision_at_20_std": 0.06876162395907931, + "nauc_precision_at_3_diff1": 0.5006947292793384, + "nauc_precision_at_3_max": 0.3504899205379297, + "nauc_precision_at_3_std": -0.0317886892590731, + "nauc_precision_at_5_diff1": 0.47950476671033915, + "nauc_precision_at_5_max": 0.36885446009820255, + "nauc_precision_at_5_std": -0.012324809862004767, + "nauc_recall_at_1000_diff1": 0.3655663317541456, + "nauc_recall_at_1000_max": 0.5292746734351961, + "nauc_recall_at_1000_std": 0.4934431913927676, + "nauc_recall_at_100_diff1": 0.4483975768333888, + "nauc_recall_at_100_max": 0.46999388192999947, + "nauc_recall_at_100_std": 0.22480444542057113, + "nauc_recall_at_10_diff1": 0.4730338188487919, + "nauc_recall_at_10_max": 0.3865349218041699, + "nauc_recall_at_10_std": 0.0218754324943048, + "nauc_recall_at_1_diff1": 0.559164843726652, + "nauc_recall_at_1_max": 0.3139070699510342, + "nauc_recall_at_1_std": -0.014207864004640072, + "nauc_recall_at_20_diff1": 0.47154875022933135, + "nauc_recall_at_20_max": 0.4140598882819984, + "nauc_recall_at_20_std": 0.0634168798351148, + "nauc_recall_at_3_diff1": 0.4884183618257877, + "nauc_recall_at_3_max": 0.3356941951765397, + "nauc_recall_at_3_std": -0.03039726760486671, + "nauc_recall_at_5_diff1": 0.4775892775554542, + "nauc_recall_at_5_max": 0.35809044526462197, + "nauc_recall_at_5_std": -0.0107654762716192, + "ndcg_at_1": 0.35839, + "ndcg_at_10": 0.49778, + "ndcg_at_100": 0.54322, + "ndcg_at_1000": 0.55462, + "ndcg_at_20": 0.51667, + "ndcg_at_3": 0.44069, + "ndcg_at_5": 0.46901, + "precision_at_1": 0.35839, + "precision_at_10": 0.07192, + "precision_at_100": 0.00973, + "precision_at_1000": 0.00109, + "precision_at_20": 0.04033, + "precision_at_3": 0.18087, + "precision_at_5": 0.12373, + "recall_at_1": 0.33423, + "recall_at_10": 0.65788, + "recall_at_100": 0.86547, + "recall_at_1000": 0.94799, + "recall_at_20": 0.72948, + "recall_at_3": 0.50371, + "recall_at_5": 0.57156 + } + ], + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.4809, + "map_at_1": 0.31116, + "map_at_10": 0.41901, + "map_at_100": 0.42816, + "map_at_1000": 0.42861, + "map_at_20": 0.42458, + "map_at_3": 0.38725, + "map_at_5": 0.40546, + "mrr_at_1": 0.32973297329732976, + "mrr_at_10": 0.43972450816510333, + "mrr_at_100": 0.4485009703297076, + "mrr_at_1000": 0.4488610630209768, + "mrr_at_20": 0.4452801024003755, + "mrr_at_3": 0.40784078407840896, + "mrr_at_5": 0.4264276427642772, + "nauc_map_at_1000_diff1": 0.49073212062094235, + "nauc_map_at_1000_max": 0.27860396518115743, + "nauc_map_at_1000_std": -0.02904696582851343, + "nauc_map_at_100_diff1": 0.4907063529066197, + "nauc_map_at_100_max": 0.2787194730073654, + "nauc_map_at_100_std": -0.028966445565383126, + "nauc_map_at_10_diff1": 0.4890094854818956, + "nauc_map_at_10_max": 0.27596464683045585, + "nauc_map_at_10_std": -0.03379749276478197, + "nauc_map_at_1_diff1": 0.5024874719502732, + "nauc_map_at_1_max": 0.2439940901965639, + "nauc_map_at_1_std": -0.041355149833804804, + "nauc_map_at_20_diff1": 0.489397156896087, + "nauc_map_at_20_max": 0.2770923106515912, + "nauc_map_at_20_std": -0.03132227455841948, + "nauc_map_at_3_diff1": 0.49255837811177783, + "nauc_map_at_3_max": 0.26783429376315204, + "nauc_map_at_3_std": -0.043307286089590016, + "nauc_map_at_5_diff1": 0.48964372076312473, + "nauc_map_at_5_max": 0.2745725014174099, + "nauc_map_at_5_std": -0.03599411983819895, + "nauc_mrr_at_1000_diff1": 0.5014421307642877, + "nauc_mrr_at_1000_max": 0.28220553013760963, + "nauc_mrr_at_1000_std": -0.03387009697909489, + "nauc_mrr_at_100_diff1": 0.5014743238957045, + "nauc_mrr_at_100_max": 0.28238295993441853, + "nauc_mrr_at_100_std": -0.033705215390150436, + "nauc_mrr_at_10_diff1": 0.49974885482066633, + "nauc_mrr_at_10_max": 0.2804298123022891, + "nauc_mrr_at_10_std": -0.03777335681254827, + "nauc_mrr_at_1_diff1": 0.5187205631526997, + "nauc_mrr_at_1_max": 0.25396264853865913, + "nauc_mrr_at_1_std": -0.04193088807899391, + "nauc_mrr_at_20_diff1": 0.5003522167254426, + "nauc_mrr_at_20_max": 0.28137566114764795, + "nauc_mrr_at_20_std": -0.035402317336889796, + "nauc_mrr_at_3_diff1": 0.504173522173663, + "nauc_mrr_at_3_max": 0.2735199125120259, + "nauc_mrr_at_3_std": -0.047022498426219285, + "nauc_mrr_at_5_diff1": 0.5005642458471171, + "nauc_mrr_at_5_max": 0.27969995494432803, + "nauc_mrr_at_5_std": -0.03937487314846688, + "nauc_ndcg_at_1000_diff1": 0.4915555155690406, + "nauc_ndcg_at_1000_max": 0.3011649131351498, + "nauc_ndcg_at_1000_std": -0.00479517350596524, + "nauc_ndcg_at_100_diff1": 0.49136701135106703, + "nauc_ndcg_at_100_max": 0.30510959584324093, + "nauc_ndcg_at_100_std": -0.0003056790825334711, + "nauc_ndcg_at_10_diff1": 0.4816909144227111, + "nauc_ndcg_at_10_max": 0.29098891887186185, + "nauc_ndcg_at_10_std": -0.025724859596075893, + "nauc_ndcg_at_1_diff1": 0.5187205631526997, + "nauc_ndcg_at_1_max": 0.25396264853865913, + "nauc_ndcg_at_1_std": -0.04193088807899391, + "nauc_ndcg_at_20_diff1": 0.48304751630951137, + "nauc_ndcg_at_20_max": 0.29551832295615277, + "nauc_ndcg_at_20_std": -0.01616241702267128, + "nauc_ndcg_at_3_diff1": 0.4908159332758585, + "nauc_ndcg_at_3_max": 0.2760709151243849, + "nauc_ndcg_at_3_std": -0.04471931935569172, + "nauc_ndcg_at_5_diff1": 0.4840473925143489, + "nauc_ndcg_at_5_max": 0.28785797992193396, + "nauc_ndcg_at_5_std": -0.030909116403820297, + "nauc_precision_at_1000_diff1": 0.1061536438237683, + "nauc_precision_at_1000_max": 0.204169234163723, + "nauc_precision_at_1000_std": 0.20484647838060505, + "nauc_precision_at_100_diff1": 0.31325587269543487, + "nauc_precision_at_100_max": 0.3654406156678911, + "nauc_precision_at_100_std": 0.23079904758754274, + "nauc_precision_at_10_diff1": 0.4182347923833404, + "nauc_precision_at_10_max": 0.3360594398532357, + "nauc_precision_at_10_std": 0.015499239952815746, + "nauc_precision_at_1_diff1": 0.5187205631526997, + "nauc_precision_at_1_max": 0.25396264853865913, + "nauc_precision_at_1_std": -0.04193088807899391, + "nauc_precision_at_20_diff1": 0.39688370691942526, + "nauc_precision_at_20_max": 0.3548445596967036, + "nauc_precision_at_20_std": 0.07330364252806168, + "nauc_precision_at_3_diff1": 0.480532877997896, + "nauc_precision_at_3_max": 0.3047410721166821, + "nauc_precision_at_3_std": -0.04380466275081102, + "nauc_precision_at_5_diff1": 0.45678096150210673, + "nauc_precision_at_5_max": 0.33489685182315454, + "nauc_precision_at_5_std": -0.006071577011863877, + "nauc_recall_at_1000_diff1": 0.3787440207406497, + "nauc_recall_at_1000_max": 0.545639563509819, + "nauc_recall_at_1000_std": 0.45281229510106885, + "nauc_recall_at_100_diff1": 0.44126828484503094, + "nauc_recall_at_100_max": 0.4713201716077075, + "nauc_recall_at_100_std": 0.27295158899030986, + "nauc_recall_at_10_diff1": 0.42435107579912035, + "nauc_recall_at_10_max": 0.3269299822640843, + "nauc_recall_at_10_std": 0.010019108980137855, + "nauc_recall_at_1_diff1": 0.5024874719502732, + "nauc_recall_at_1_max": 0.2439940901965639, + "nauc_recall_at_1_std": -0.041355149833804804, + "nauc_recall_at_20_diff1": 0.41210041800891617, + "nauc_recall_at_20_max": 0.3509266713055881, + "nauc_recall_at_20_std": 0.06388334851612927, + "nauc_recall_at_3_diff1": 0.4709649932229704, + "nauc_recall_at_3_max": 0.292024322704373, + "nauc_recall_at_3_std": -0.04294418609930153, + "nauc_recall_at_5_diff1": 0.4445029920289486, + "nauc_recall_at_5_max": 0.31665446054702767, + "nauc_recall_at_5_std": -0.009965031777792056, + "ndcg_at_1": 0.32973, + "ndcg_at_10": 0.4809, + "ndcg_at_100": 0.52429, + "ndcg_at_1000": 0.53611, + "ndcg_at_20": 0.50044, + "ndcg_at_3": 0.41681, + "ndcg_at_5": 0.44909, + "precision_at_1": 0.32973, + "precision_at_10": 0.07121, + "precision_at_100": 0.00947, + "precision_at_1000": 0.00107, + "precision_at_20": 0.03986, + "precision_at_3": 0.17212, + "precision_at_5": 0.12052, + "recall_at_1": 0.31116, + "recall_at_10": 0.65379, + "recall_at_100": 0.84943, + "recall_at_1000": 0.93832, + "recall_at_20": 0.72784, + "recall_at_3": 0.48157, + "recall_at_5": 0.55865 + } + ], + "train": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.38431, + "map_at_1": 0.22546, + "map_at_10": 0.32513, + "map_at_100": 0.33708, + "map_at_1000": 0.33778, + "map_at_20": 0.33197, + "map_at_3": 0.29484, + "map_at_5": 0.312, + "mrr_at_1": 0.24662599034696295, + "mrr_at_10": 0.34958735723896506, + "mrr_at_100": 0.36090128669603605, + "mrr_at_1000": 0.36143137214198995, + "mrr_at_20": 0.3562764988370791, + "mrr_at_3": 0.31911635248761083, + "mrr_at_5": 0.3364467413410986, + "nauc_map_at_1000_diff1": 0.44203093505237034, + "nauc_map_at_1000_max": 0.2904505178289206, + "nauc_map_at_1000_std": -0.02935834243148873, + "nauc_map_at_100_diff1": 0.44206120272073435, + "nauc_map_at_100_max": 0.290634056956638, + "nauc_map_at_100_std": -0.029453776012505945, + "nauc_map_at_10_diff1": 0.4407679076293158, + "nauc_map_at_10_max": 0.2870060446965365, + "nauc_map_at_10_std": -0.034838271179798645, + "nauc_map_at_1_diff1": 0.45377876037871984, + "nauc_map_at_1_max": 0.2511307127628015, + "nauc_map_at_1_std": -0.0414310878674339, + "nauc_map_at_20_diff1": 0.4416512739405299, + "nauc_map_at_20_max": 0.2893819470204192, + "nauc_map_at_20_std": -0.03243344553581258, + "nauc_map_at_3_diff1": 0.4371652429750249, + "nauc_map_at_3_max": 0.27470631366494397, + "nauc_map_at_3_std": -0.04148811612305109, + "nauc_map_at_5_diff1": 0.43909113446174686, + "nauc_map_at_5_max": 0.2819045374188352, + "nauc_map_at_5_std": -0.03843428636603287, + "nauc_mrr_at_1000_diff1": 0.4506198350320849, + "nauc_mrr_at_1000_max": 0.29244792644977563, + "nauc_mrr_at_1000_std": -0.03686179374981812, + "nauc_mrr_at_100_diff1": 0.45063001772040817, + "nauc_mrr_at_100_max": 0.29267500732470514, + "nauc_mrr_at_100_std": -0.03676978346122215, + "nauc_mrr_at_10_diff1": 0.4494294540285445, + "nauc_mrr_at_10_max": 0.29052685172003667, + "nauc_mrr_at_10_std": -0.040580743470482336, + "nauc_mrr_at_1_diff1": 0.4673839293610825, + "nauc_mrr_at_1_max": 0.26307943131391953, + "nauc_mrr_at_1_std": -0.04362658743740279, + "nauc_mrr_at_20_diff1": 0.45022152152373435, + "nauc_mrr_at_20_max": 0.2920824347663815, + "nauc_mrr_at_20_std": -0.038804641599828577, + "nauc_mrr_at_3_diff1": 0.44675338464879283, + "nauc_mrr_at_3_max": 0.28120222808693845, + "nauc_mrr_at_3_std": -0.04539541108316358, + "nauc_mrr_at_5_diff1": 0.4479969516503765, + "nauc_mrr_at_5_max": 0.2869337482663382, + "nauc_mrr_at_5_std": -0.04305936580156582, + "nauc_ndcg_at_1000_diff1": 0.4450047377663919, + "nauc_ndcg_at_1000_max": 0.31403191694614363, + "nauc_ndcg_at_1000_std": -0.0027344717728348235, + "nauc_ndcg_at_100_diff1": 0.4459065668576569, + "nauc_ndcg_at_100_max": 0.3204917945868563, + "nauc_ndcg_at_100_std": -0.00022168732681981113, + "nauc_ndcg_at_10_diff1": 0.4403116219829576, + "nauc_ndcg_at_10_max": 0.3039570300876629, + "nauc_ndcg_at_10_std": -0.03004670400000527, + "nauc_ndcg_at_1_diff1": 0.4674568770896304, + "nauc_ndcg_at_1_max": 0.2631639273070747, + "nauc_ndcg_at_1_std": -0.04351961499716811, + "nauc_ndcg_at_20_diff1": 0.4431969864774668, + "nauc_ndcg_at_20_max": 0.31200876249901693, + "nauc_ndcg_at_20_std": -0.021667681260148824, + "nauc_ndcg_at_3_diff1": 0.43408495083118176, + "nauc_ndcg_at_3_max": 0.2810997089863219, + "nauc_ndcg_at_3_std": -0.04318871947280584, + "nauc_ndcg_at_5_diff1": 0.43675657395429135, + "nauc_ndcg_at_5_max": 0.2928243435235781, + "nauc_ndcg_at_5_std": -0.03797036181553665, + "nauc_precision_at_1000_diff1": -0.009701766132387591, + "nauc_precision_at_1000_max": 0.09273993408537871, + "nauc_precision_at_1000_std": 0.18330110524342091, + "nauc_precision_at_100_diff1": 0.21089648871839442, + "nauc_precision_at_100_max": 0.3025506739012124, + "nauc_precision_at_100_std": 0.19760012979970706, + "nauc_precision_at_10_diff1": 0.39978008334698417, + "nauc_precision_at_10_max": 0.34567175928277133, + "nauc_precision_at_10_std": -0.0029372364840954492, + "nauc_precision_at_1_diff1": 0.4674568770896304, + "nauc_precision_at_1_max": 0.2631639273070747, + "nauc_precision_at_1_std": -0.04351961499716811, + "nauc_precision_at_20_diff1": 0.37171960673928006, + "nauc_precision_at_20_max": 0.3578739905564068, + "nauc_precision_at_20_std": 0.03847026650921241, + "nauc_precision_at_3_diff1": 0.4199765754509781, + "nauc_precision_at_3_max": 0.30481164516670983, + "nauc_precision_at_3_std": -0.043443253008935055, + "nauc_precision_at_5_diff1": 0.41328519881945674, + "nauc_precision_at_5_max": 0.3248223849334302, + "nauc_precision_at_5_std": -0.030213049417602042, + "nauc_recall_at_1000_diff1": 0.36268121068380327, + "nauc_recall_at_1000_max": 0.5352280225392065, + "nauc_recall_at_1000_std": 0.5363470750685398, + "nauc_recall_at_100_diff1": 0.4181057276037194, + "nauc_recall_at_100_max": 0.4765876512143782, + "nauc_recall_at_100_std": 0.24575279116465742, + "nauc_recall_at_10_diff1": 0.4138651067229952, + "nauc_recall_at_10_max": 0.3373411923439824, + "nauc_recall_at_10_std": -0.008172381612455301, + "nauc_recall_at_1_diff1": 0.45377876037871984, + "nauc_recall_at_1_max": 0.2511307127628015, + "nauc_recall_at_1_std": -0.0414310878674339, + "nauc_recall_at_20_diff1": 0.41683851087708707, + "nauc_recall_at_20_max": 0.37018847091041124, + "nauc_recall_at_20_std": 0.028919962032348905, + "nauc_recall_at_3_diff1": 0.41143915619124266, + "nauc_recall_at_3_max": 0.2879126339358578, + "nauc_recall_at_3_std": -0.04299904565599763, + "nauc_recall_at_5_diff1": 0.41123484821777284, + "nauc_recall_at_5_max": 0.30886900216513435, + "nauc_recall_at_5_std": -0.03171586406884398, + "ndcg_at_1": 0.24661, + "ndcg_at_10": 0.38431, + "ndcg_at_100": 0.44336, + "ndcg_at_1000": 0.46004, + "ndcg_at_20": 0.40849, + "ndcg_at_3": 0.32315, + "ndcg_at_5": 0.3533, + "precision_at_1": 0.24661, + "precision_at_10": 0.06141, + "precision_at_100": 0.00956, + "precision_at_1000": 0.00113, + "precision_at_20": 0.03627, + "precision_at_3": 0.14045, + "precision_at_5": 0.10088, + "recall_at_1": 0.22546, + "recall_at_10": 0.54479, + "recall_at_100": 0.81412, + "recall_at_1000": 0.93629, + "recall_at_20": 0.63638, + "recall_at_3": 0.38015, + "recall_at_5": 0.45222 + } + ] + }, + "task_name": "FEVER" +} \ No newline at end of file diff --git a/results/bm25s/0_1_10/FiQA2018.json b/results/bm25s/0_1_10/FiQA2018.json new file mode 100644 index 0000000000..fc64367a86 --- /dev/null +++ b/results/bm25s/0_1_10/FiQA2018.json @@ -0,0 +1,456 @@ +{ + "dataset_revision": "27a168819829fe9bcd655c2df245fb19452e8e06", + "evaluation_time": 42.59879207611084, + "kg_co2_emissions": null, + "mteb_version": "1.12.22", + "scores": { + "dev": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.2592, + "map_at_1": 0.13255, + "map_at_10": 0.20296, + "map_at_100": 0.21674, + "map_at_1000": 0.21831, + "map_at_20": 0.2103, + "map_at_3": 0.18024, + "map_at_5": 0.19329, + "mrr_at_1": 0.236, + "mrr_at_10": 0.3162626984126984, + "mrr_at_100": 0.326560123936558, + "mrr_at_1000": 0.3271669142769114, + "mrr_at_20": 0.32270843559605167, + "mrr_at_3": 0.29433333333333334, + "mrr_at_5": 0.3061333333333333, + "nauc_map_at_1000_diff1": 0.4706127089094157, + "nauc_map_at_1000_max": 0.3530042124867514, + "nauc_map_at_1000_std": 0.1801837601880982, + "nauc_map_at_100_diff1": 0.47060696566991816, + "nauc_map_at_100_max": 0.3516743907744571, + "nauc_map_at_100_std": 0.17932477362397348, + "nauc_map_at_10_diff1": 0.4674503178628199, + "nauc_map_at_10_max": 0.3362890934461591, + "nauc_map_at_10_std": 0.16145709045781942, + "nauc_map_at_1_diff1": 0.5547395776466849, + "nauc_map_at_1_max": 0.31761531604158216, + "nauc_map_at_1_std": 0.14454574959461064, + "nauc_map_at_20_diff1": 0.4681034062408604, + "nauc_map_at_20_max": 0.34526534933606007, + "nauc_map_at_20_std": 0.17138576564917826, + "nauc_map_at_3_diff1": 0.47256646056858653, + "nauc_map_at_3_max": 0.3200344013399651, + "nauc_map_at_3_std": 0.1525832111961707, + "nauc_map_at_5_diff1": 0.4697249017865003, + "nauc_map_at_5_max": 0.33367542434108416, + "nauc_map_at_5_std": 0.15673269198317247, + "nauc_mrr_at_1000_diff1": 0.4670659620311787, + "nauc_mrr_at_1000_max": 0.39003063904364055, + "nauc_mrr_at_1000_std": 0.19483102289488116, + "nauc_mrr_at_100_diff1": 0.46703516825027847, + "nauc_mrr_at_100_max": 0.3899973939646035, + "nauc_mrr_at_100_std": 0.19493864249536397, + "nauc_mrr_at_10_diff1": 0.46478060936618804, + "nauc_mrr_at_10_max": 0.386418992347399, + "nauc_mrr_at_10_std": 0.18875756641066355, + "nauc_mrr_at_1_diff1": 0.5414029074363511, + "nauc_mrr_at_1_max": 0.4101699926982194, + "nauc_mrr_at_1_std": 0.20822144309740348, + "nauc_mrr_at_20_diff1": 0.465534922107964, + "nauc_mrr_at_20_max": 0.38802101954489543, + "nauc_mrr_at_20_std": 0.19102739528359958, + "nauc_mrr_at_3_diff1": 0.46370641331457757, + "nauc_mrr_at_3_max": 0.38379016612286077, + "nauc_mrr_at_3_std": 0.188230546150994, + "nauc_mrr_at_5_diff1": 0.4641390154165986, + "nauc_mrr_at_5_max": 0.38697041287908657, + "nauc_mrr_at_5_std": 0.18721890890042236, + "nauc_ndcg_at_1000_diff1": 0.45927843555467723, + "nauc_ndcg_at_1000_max": 0.39917255698199294, + "nauc_ndcg_at_1000_std": 0.23233876722058266, + "nauc_ndcg_at_100_diff1": 0.45673807015067713, + "nauc_ndcg_at_100_max": 0.3837545087145896, + "nauc_ndcg_at_100_std": 0.2252517924584521, + "nauc_ndcg_at_10_diff1": 0.44381648883584635, + "nauc_ndcg_at_10_max": 0.34790294325586374, + "nauc_ndcg_at_10_std": 0.17166109874399138, + "nauc_ndcg_at_1_diff1": 0.5414029074363511, + "nauc_ndcg_at_1_max": 0.4101699926982194, + "nauc_ndcg_at_1_std": 0.20822144309740348, + "nauc_ndcg_at_20_diff1": 0.4477343120537464, + "nauc_ndcg_at_20_max": 0.36199515596666965, + "nauc_ndcg_at_20_std": 0.18916515674533796, + "nauc_ndcg_at_3_diff1": 0.43958716367894246, + "nauc_ndcg_at_3_max": 0.35121675205653496, + "nauc_ndcg_at_3_std": 0.1701078514820257, + "nauc_ndcg_at_5_diff1": 0.4450279157321356, + "nauc_ndcg_at_5_max": 0.35493010852407686, + "nauc_ndcg_at_5_std": 0.1702207229610164, + "nauc_precision_at_1000_diff1": 0.10472887577461688, + "nauc_precision_at_1000_max": 0.2980535181187962, + "nauc_precision_at_1000_std": 0.22219503729289158, + "nauc_precision_at_100_diff1": 0.19776592641740293, + "nauc_precision_at_100_max": 0.3599818098288429, + "nauc_precision_at_100_std": 0.2752782341369501, + "nauc_precision_at_10_diff1": 0.28087804366236235, + "nauc_precision_at_10_max": 0.37168616459196735, + "nauc_precision_at_10_std": 0.18736894171606697, + "nauc_precision_at_1_diff1": 0.5414029074363511, + "nauc_precision_at_1_max": 0.4101699926982194, + "nauc_precision_at_1_std": 0.20822144309740348, + "nauc_precision_at_20_diff1": 0.25550774659479014, + "nauc_precision_at_20_max": 0.38823248014485, + "nauc_precision_at_20_std": 0.23147581375252038, + "nauc_precision_at_3_diff1": 0.3392691300622539, + "nauc_precision_at_3_max": 0.3852895298697201, + "nauc_precision_at_3_std": 0.1874828986340876, + "nauc_precision_at_5_diff1": 0.3313908708859799, + "nauc_precision_at_5_max": 0.39875216028588994, + "nauc_precision_at_5_std": 0.1874413082846425, + "nauc_recall_at_1000_diff1": 0.29751655187772, + "nauc_recall_at_1000_max": 0.4367478095745362, + "nauc_recall_at_1000_std": 0.3958089659024556, + "nauc_recall_at_100_diff1": 0.32421318097263985, + "nauc_recall_at_100_max": 0.34275976810400155, + "nauc_recall_at_100_std": 0.30930444244331173, + "nauc_recall_at_10_diff1": 0.34693197148303734, + "nauc_recall_at_10_max": 0.2793985405095117, + "nauc_recall_at_10_std": 0.1461946372882578, + "nauc_recall_at_1_diff1": 0.5547395776466849, + "nauc_recall_at_1_max": 0.31761531604158216, + "nauc_recall_at_1_std": 0.14454574959461064, + "nauc_recall_at_20_diff1": 0.338279828865901, + "nauc_recall_at_20_max": 0.30589668008983945, + "nauc_recall_at_20_std": 0.191283211796953, + "nauc_recall_at_3_diff1": 0.39034135224179256, + "nauc_recall_at_3_max": 0.28782173546248785, + "nauc_recall_at_3_std": 0.14005787438079884, + "nauc_recall_at_5_diff1": 0.36688220167255614, + "nauc_recall_at_5_max": 0.29784819982904026, + "nauc_recall_at_5_std": 0.13842361348788487, + "ndcg_at_1": 0.236, + "ndcg_at_10": 0.2592, + "ndcg_at_100": 0.31959, + "ndcg_at_1000": 0.35353, + "ndcg_at_20": 0.28176, + "ndcg_at_3": 0.2318, + "ndcg_at_5": 0.24215, + "precision_at_1": 0.236, + "precision_at_10": 0.065, + "precision_at_100": 0.01262, + "precision_at_1000": 0.00184, + "precision_at_20": 0.0416, + "precision_at_3": 0.142, + "precision_at_5": 0.1036, + "recall_at_1": 0.13255, + "recall_at_10": 0.31435, + "recall_at_100": 0.54946, + "recall_at_1000": 0.76526, + "recall_at_20": 0.38705, + "recall_at_3": 0.21997, + "recall_at_5": 0.26155 + } + ], + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.25139, + "map_at_1": 0.11426, + "map_at_10": 0.19003, + "map_at_100": 0.20401, + "map_at_1000": 0.20564, + "map_at_20": 0.19765, + "map_at_3": 0.16345, + "map_at_5": 0.17629, + "mrr_at_1": 0.22993827160493827, + "mrr_at_10": 0.3074998775230256, + "mrr_at_100": 0.31834918350235314, + "mrr_at_1000": 0.3189151471924152, + "mrr_at_20": 0.314273373250268, + "mrr_at_3": 0.2836934156378601, + "mrr_at_5": 0.297582304526749, + "nauc_map_at_1000_diff1": 0.33548132847127243, + "nauc_map_at_1000_max": 0.30203502530163623, + "nauc_map_at_1000_std": 0.08667832884668883, + "nauc_map_at_100_diff1": 0.33550660963754375, + "nauc_map_at_100_max": 0.3016241381976774, + "nauc_map_at_100_std": 0.08624108112077933, + "nauc_map_at_10_diff1": 0.33764260813132285, + "nauc_map_at_10_max": 0.2966630749996222, + "nauc_map_at_10_std": 0.07432107038125393, + "nauc_map_at_1_diff1": 0.38818529229610627, + "nauc_map_at_1_max": 0.26058083007080934, + "nauc_map_at_1_std": 0.04841687084526099, + "nauc_map_at_20_diff1": 0.33477866014196195, + "nauc_map_at_20_max": 0.30035851057263807, + "nauc_map_at_20_std": 0.08236803774578869, + "nauc_map_at_3_diff1": 0.3529826544662004, + "nauc_map_at_3_max": 0.28326352958521617, + "nauc_map_at_3_std": 0.05093698916379205, + "nauc_map_at_5_diff1": 0.3451918509049883, + "nauc_map_at_5_max": 0.2857818662998791, + "nauc_map_at_5_std": 0.06194439934953413, + "nauc_mrr_at_1000_diff1": 0.36288788118037135, + "nauc_mrr_at_1000_max": 0.3004981986942584, + "nauc_mrr_at_1000_std": 0.06722182156014167, + "nauc_mrr_at_100_diff1": 0.36268609436947785, + "nauc_mrr_at_100_max": 0.30050388143972595, + "nauc_mrr_at_100_std": 0.06747023282377294, + "nauc_mrr_at_10_diff1": 0.36332409878535465, + "nauc_mrr_at_10_max": 0.3023812257431008, + "nauc_mrr_at_10_std": 0.06639577061264093, + "nauc_mrr_at_1_diff1": 0.4272273703958737, + "nauc_mrr_at_1_max": 0.29491623182546, + "nauc_mrr_at_1_std": 0.04308100793993664, + "nauc_mrr_at_20_diff1": 0.3632020492671718, + "nauc_mrr_at_20_max": 0.3007955195190496, + "nauc_mrr_at_20_std": 0.0665750325152591, + "nauc_mrr_at_3_diff1": 0.3741448567511275, + "nauc_mrr_at_3_max": 0.295175193060477, + "nauc_mrr_at_3_std": 0.047824375528965325, + "nauc_mrr_at_5_diff1": 0.37074618072678106, + "nauc_mrr_at_5_max": 0.2998775822345696, + "nauc_mrr_at_5_std": 0.0635737801434362, + "nauc_ndcg_at_1000_diff1": 0.31118716902517674, + "nauc_ndcg_at_1000_max": 0.32265806528475005, + "nauc_ndcg_at_1000_std": 0.12541706889075654, + "nauc_ndcg_at_100_diff1": 0.3146405076555731, + "nauc_ndcg_at_100_max": 0.31839410648209976, + "nauc_ndcg_at_100_std": 0.12472417901902241, + "nauc_ndcg_at_10_diff1": 0.31887425574997935, + "nauc_ndcg_at_10_max": 0.3100527747490141, + "nauc_ndcg_at_10_std": 0.09086179353837337, + "nauc_ndcg_at_1_diff1": 0.4272273703958737, + "nauc_ndcg_at_1_max": 0.29491623182546, + "nauc_ndcg_at_1_std": 0.04308100793993664, + "nauc_ndcg_at_20_diff1": 0.3114358966372274, + "nauc_ndcg_at_20_max": 0.3121882707520853, + "nauc_ndcg_at_20_std": 0.10528818112551663, + "nauc_ndcg_at_3_diff1": 0.33754587112213535, + "nauc_ndcg_at_3_max": 0.2822217263347099, + "nauc_ndcg_at_3_std": 0.04335091673628977, + "nauc_ndcg_at_5_diff1": 0.3341651398682917, + "nauc_ndcg_at_5_max": 0.2921614972510362, + "nauc_ndcg_at_5_std": 0.06974207408839966, + "nauc_precision_at_1000_diff1": 0.045503938454060644, + "nauc_precision_at_1000_max": 0.14095624820566788, + "nauc_precision_at_1000_std": 0.09682977529314721, + "nauc_precision_at_100_diff1": 0.13834891994552553, + "nauc_precision_at_100_max": 0.22112682152033192, + "nauc_precision_at_100_std": 0.15511004752915608, + "nauc_precision_at_10_diff1": 0.2528610560520703, + "nauc_precision_at_10_max": 0.30337970161146616, + "nauc_precision_at_10_std": 0.11337486174702292, + "nauc_precision_at_1_diff1": 0.4272273703958737, + "nauc_precision_at_1_max": 0.29491623182546, + "nauc_precision_at_1_std": 0.04308100793993664, + "nauc_precision_at_20_diff1": 0.21087164381632248, + "nauc_precision_at_20_max": 0.28427960675464087, + "nauc_precision_at_20_std": 0.13982829951049214, + "nauc_precision_at_3_diff1": 0.31475172418762787, + "nauc_precision_at_3_max": 0.2800841028459411, + "nauc_precision_at_3_std": 0.02948685601949665, + "nauc_precision_at_5_diff1": 0.2929473742406975, + "nauc_precision_at_5_max": 0.2833358028246732, + "nauc_precision_at_5_std": 0.07689631564150524, + "nauc_recall_at_1000_diff1": 0.08965370535414983, + "nauc_recall_at_1000_max": 0.3283604264242104, + "nauc_recall_at_1000_std": 0.2940846404698579, + "nauc_recall_at_100_diff1": 0.172348504025239, + "nauc_recall_at_100_max": 0.2965507837621931, + "nauc_recall_at_100_std": 0.22770333753950625, + "nauc_recall_at_10_diff1": 0.20559167842480186, + "nauc_recall_at_10_max": 0.28887460144634614, + "nauc_recall_at_10_std": 0.11944187726387592, + "nauc_recall_at_1_diff1": 0.38818529229610627, + "nauc_recall_at_1_max": 0.26058083007080934, + "nauc_recall_at_1_std": 0.04841687084526099, + "nauc_recall_at_20_diff1": 0.17483382976790707, + "nauc_recall_at_20_max": 0.2803509605927133, + "nauc_recall_at_20_std": 0.15089125075193854, + "nauc_recall_at_3_diff1": 0.27812134908988995, + "nauc_recall_at_3_max": 0.2693155875923942, + "nauc_recall_at_3_std": 0.056613375482027294, + "nauc_recall_at_5_diff1": 0.25653114422731627, + "nauc_recall_at_5_max": 0.26934952816552693, + "nauc_recall_at_5_std": 0.08688352380887347, + "ndcg_at_1": 0.22994, + "ndcg_at_10": 0.25139, + "ndcg_at_100": 0.31424, + "ndcg_at_1000": 0.34888, + "ndcg_at_20": 0.27557, + "ndcg_at_3": 0.21769, + "ndcg_at_5": 0.2274, + "precision_at_1": 0.22994, + "precision_at_10": 0.06991, + "precision_at_100": 0.01326, + "precision_at_1000": 0.00196, + "precision_at_20": 0.04414, + "precision_at_3": 0.14558, + "precision_at_5": 0.10741, + "recall_at_1": 0.11426, + "recall_at_10": 0.31779, + "recall_at_100": 0.55933, + "recall_at_1000": 0.77165, + "recall_at_20": 0.39538, + "recall_at_3": 0.20274, + "recall_at_5": 0.24542 + } + ], + "train": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.24794, + "map_at_1": 0.12213, + "map_at_10": 0.19158, + "map_at_100": 0.20473, + "map_at_1000": 0.20639, + "map_at_20": 0.19857, + "map_at_3": 0.16601, + "map_at_5": 0.17943, + "mrr_at_1": 0.22963636363636364, + "mrr_at_10": 0.30032142857142763, + "mrr_at_100": 0.30988947549185336, + "mrr_at_1000": 0.31061177445507243, + "mrr_at_20": 0.30572431667684846, + "mrr_at_3": 0.27869696969696905, + "mrr_at_5": 0.29003333333333325, + "nauc_map_at_1000_diff1": 0.38363230530218034, + "nauc_map_at_1000_max": 0.3331628097995852, + "nauc_map_at_1000_std": 0.1403787649723463, + "nauc_map_at_100_diff1": 0.38363654733232344, + "nauc_map_at_100_max": 0.33286204181062323, + "nauc_map_at_100_std": 0.13975503217119462, + "nauc_map_at_10_diff1": 0.3824123008223932, + "nauc_map_at_10_max": 0.32506463197809454, + "nauc_map_at_10_std": 0.12807971708177393, + "nauc_map_at_1_diff1": 0.4306150954748461, + "nauc_map_at_1_max": 0.30576305181378444, + "nauc_map_at_1_std": 0.11122744532097459, + "nauc_map_at_20_diff1": 0.38354172334090836, + "nauc_map_at_20_max": 0.3299594105016288, + "nauc_map_at_20_std": 0.13456724119419788, + "nauc_map_at_3_diff1": 0.3919461710499075, + "nauc_map_at_3_max": 0.3157918044297698, + "nauc_map_at_3_std": 0.11521232986707576, + "nauc_map_at_5_diff1": 0.38647109273817704, + "nauc_map_at_5_max": 0.32103756934684374, + "nauc_map_at_5_std": 0.12151423640446292, + "nauc_mrr_at_1000_diff1": 0.3968067800551367, + "nauc_mrr_at_1000_max": 0.32186890630140175, + "nauc_mrr_at_1000_std": 0.11909664165059941, + "nauc_mrr_at_100_diff1": 0.3965636583204436, + "nauc_mrr_at_100_max": 0.32187874433275526, + "nauc_mrr_at_100_std": 0.11919912588417539, + "nauc_mrr_at_10_diff1": 0.3973558213474224, + "nauc_mrr_at_10_max": 0.31920699548337267, + "nauc_mrr_at_10_std": 0.11439991689700359, + "nauc_mrr_at_1_diff1": 0.4512056670277103, + "nauc_mrr_at_1_max": 0.32410243431209207, + "nauc_mrr_at_1_std": 0.09945160750515963, + "nauc_mrr_at_20_diff1": 0.3968434532129195, + "nauc_mrr_at_20_max": 0.3210507719917609, + "nauc_mrr_at_20_std": 0.11755615614371845, + "nauc_mrr_at_3_diff1": 0.40523006338873213, + "nauc_mrr_at_3_max": 0.31769317709962475, + "nauc_mrr_at_3_std": 0.10828257196275703, + "nauc_mrr_at_5_diff1": 0.3994699186633496, + "nauc_mrr_at_5_max": 0.3175893522378336, + "nauc_mrr_at_5_std": 0.11050217380070265, + "nauc_ndcg_at_1000_diff1": 0.3668480360270626, + "nauc_ndcg_at_1000_max": 0.35139120981554306, + "nauc_ndcg_at_1000_std": 0.18163369186613512, + "nauc_ndcg_at_100_diff1": 0.3665501751951809, + "nauc_ndcg_at_100_max": 0.34921498615238955, + "nauc_ndcg_at_100_std": 0.17673953408194865, + "nauc_ndcg_at_10_diff1": 0.3673809195798603, + "nauc_ndcg_at_10_max": 0.32678277489046337, + "nauc_ndcg_at_10_std": 0.136570733620268, + "nauc_ndcg_at_1_diff1": 0.4512056670277103, + "nauc_ndcg_at_1_max": 0.32410243431209207, + "nauc_ndcg_at_1_std": 0.09945160750515963, + "nauc_ndcg_at_20_diff1": 0.36780681673922294, + "nauc_ndcg_at_20_max": 0.33737802734460615, + "nauc_ndcg_at_20_std": 0.15314788561331957, + "nauc_ndcg_at_3_diff1": 0.38534402946412166, + "nauc_ndcg_at_3_max": 0.3191433402104887, + "nauc_ndcg_at_3_std": 0.11455311618484501, + "nauc_ndcg_at_5_diff1": 0.3762538030684215, + "nauc_ndcg_at_5_max": 0.32227162185513775, + "nauc_ndcg_at_5_std": 0.1235605192718476, + "nauc_precision_at_1000_diff1": 0.08436607816639592, + "nauc_precision_at_1000_max": 0.16118834945391652, + "nauc_precision_at_1000_std": 0.13237880575358893, + "nauc_precision_at_100_diff1": 0.18892405035642484, + "nauc_precision_at_100_max": 0.26969877533831754, + "nauc_precision_at_100_std": 0.19705764917728286, + "nauc_precision_at_10_diff1": 0.27140179952998705, + "nauc_precision_at_10_max": 0.30758482734806064, + "nauc_precision_at_10_std": 0.14345009696335187, + "nauc_precision_at_1_diff1": 0.4512056670277103, + "nauc_precision_at_1_max": 0.32410243431209207, + "nauc_precision_at_1_std": 0.09945160750515963, + "nauc_precision_at_20_diff1": 0.25278828013412485, + "nauc_precision_at_20_max": 0.306435625738575, + "nauc_precision_at_20_std": 0.16771842991682195, + "nauc_precision_at_3_diff1": 0.3448076282485792, + "nauc_precision_at_3_max": 0.32263497777764255, + "nauc_precision_at_3_std": 0.11651642265909799, + "nauc_precision_at_5_diff1": 0.31146601861629947, + "nauc_precision_at_5_max": 0.31954621724118587, + "nauc_precision_at_5_std": 0.13036611895726768, + "nauc_recall_at_1000_diff1": 0.20604759543468787, + "nauc_recall_at_1000_max": 0.34849009911806395, + "nauc_recall_at_1000_std": 0.3450849986092287, + "nauc_recall_at_100_diff1": 0.24380700669348881, + "nauc_recall_at_100_max": 0.3315034738830664, + "nauc_recall_at_100_std": 0.26361835708684667, + "nauc_recall_at_10_diff1": 0.27831240017981773, + "nauc_recall_at_10_max": 0.28845675775286284, + "nauc_recall_at_10_std": 0.14355077244512646, + "nauc_recall_at_1_diff1": 0.4306150954748461, + "nauc_recall_at_1_max": 0.30576305181378444, + "nauc_recall_at_1_std": 0.11122744532097459, + "nauc_recall_at_20_diff1": 0.2698907784944336, + "nauc_recall_at_20_max": 0.30725057628003527, + "nauc_recall_at_20_std": 0.18398313693804977, + "nauc_recall_at_3_diff1": 0.3315896878226532, + "nauc_recall_at_3_max": 0.2919282926331545, + "nauc_recall_at_3_std": 0.11471379086293128, + "nauc_recall_at_5_diff1": 0.30807941524833465, + "nauc_recall_at_5_max": 0.2907598431886408, + "nauc_recall_at_5_std": 0.12409365858007287, + "ndcg_at_1": 0.22964, + "ndcg_at_10": 0.24794, + "ndcg_at_100": 0.30524, + "ndcg_at_1000": 0.3407, + "ndcg_at_20": 0.26885, + "ndcg_at_3": 0.21505, + "ndcg_at_5": 0.2262, + "precision_at_1": 0.22964, + "precision_at_10": 0.06667, + "precision_at_100": 0.01231, + "precision_at_1000": 0.00184, + "precision_at_20": 0.04162, + "precision_at_3": 0.13776, + "precision_at_5": 0.1024, + "recall_at_1": 0.12213, + "recall_at_10": 0.30698, + "recall_at_100": 0.5286, + "recall_at_1000": 0.74905, + "recall_at_20": 0.37415, + "recall_at_3": 0.20252, + "recall_at_5": 0.24422 + } + ] + }, + "task_name": "FiQA2018" +} \ No newline at end of file diff --git a/results/bm25s/0_1_10/HotpotQA.json b/results/bm25s/0_1_10/HotpotQA.json new file mode 100644 index 0000000000..7823306b48 --- /dev/null +++ b/results/bm25s/0_1_10/HotpotQA.json @@ -0,0 +1,456 @@ +{ + "dataset_revision": "ab518f4d6fcca38d87c25209f94beba119d02014", + "evaluation_time": 4223.513247013092, + "kg_co2_emissions": null, + "mteb_version": "1.12.22", + "scores": { + "dev": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.59451, + "map_at_1": 0.3412, + "map_at_10": 0.50229, + "map_at_100": 0.51285, + "map_at_1000": 0.51359, + "map_at_20": 0.50875, + "map_at_3": 0.46925, + "map_at_5": 0.48837, + "mrr_at_1": 0.68221039104094, + "mrr_at_10": 0.7523797867473301, + "mrr_at_100": 0.7557239100836104, + "mrr_at_1000": 0.7558484707271663, + "mrr_at_20": 0.754546088380347, + "mrr_at_3": 0.7370723945902967, + "mrr_at_5": 0.7464261673092236, + "nauc_map_at_1000_diff1": 0.4627196181270465, + "nauc_map_at_1000_max": 0.47591258114500506, + "nauc_map_at_1000_std": 0.26021189026460667, + "nauc_map_at_100_diff1": 0.4625687899416726, + "nauc_map_at_100_max": 0.4757630022257754, + "nauc_map_at_100_std": 0.2598919074534889, + "nauc_map_at_10_diff1": 0.46233404649664883, + "nauc_map_at_10_max": 0.4733890219198898, + "nauc_map_at_10_std": 0.2530720265047738, + "nauc_map_at_1_diff1": 0.7959089560806106, + "nauc_map_at_1_max": 0.5833537045527906, + "nauc_map_at_1_std": 0.2247460991649088, + "nauc_map_at_20_diff1": 0.46230759872174454, + "nauc_map_at_20_max": 0.4743029195373532, + "nauc_map_at_20_std": 0.2564680618494724, + "nauc_map_at_3_diff1": 0.47315207690994404, + "nauc_map_at_3_max": 0.4716249362886286, + "nauc_map_at_3_std": 0.24426626856420228, + "nauc_map_at_5_diff1": 0.46754362246162934, + "nauc_map_at_5_max": 0.47277842718821933, + "nauc_map_at_5_std": 0.2483765731290516, + "nauc_mrr_at_1000_diff1": 0.7818650969107881, + "nauc_mrr_at_1000_max": 0.5943124725004106, + "nauc_mrr_at_1000_std": 0.23283726789981912, + "nauc_mrr_at_100_diff1": 0.7818405834353004, + "nauc_mrr_at_100_max": 0.5943880234102982, + "nauc_mrr_at_100_std": 0.2329735920029303, + "nauc_mrr_at_10_diff1": 0.7813600237289062, + "nauc_mrr_at_10_max": 0.5942175959453722, + "nauc_mrr_at_10_std": 0.23161098383488907, + "nauc_mrr_at_1_diff1": 0.7962904160261535, + "nauc_mrr_at_1_max": 0.5831878819830072, + "nauc_mrr_at_1_std": 0.22427650044318287, + "nauc_mrr_at_20_diff1": 0.7813794509029378, + "nauc_mrr_at_20_max": 0.5940996482824483, + "nauc_mrr_at_20_std": 0.23221937867606088, + "nauc_mrr_at_3_diff1": 0.7805378529321749, + "nauc_mrr_at_3_max": 0.5905859343927085, + "nauc_mrr_at_3_std": 0.22521806023614133, + "nauc_mrr_at_5_diff1": 0.7814285043751165, + "nauc_mrr_at_5_max": 0.5927745409257315, + "nauc_mrr_at_5_std": 0.2287980167462581, + "nauc_ndcg_at_1000_diff1": 0.5025945367873053, + "nauc_ndcg_at_1000_max": 0.5056795406405934, + "nauc_ndcg_at_1000_std": 0.2909452202255527, + "nauc_ndcg_at_100_diff1": 0.49887477756825427, + "nauc_ndcg_at_100_max": 0.5028791429697356, + "nauc_ndcg_at_100_std": 0.28613247481299287, + "nauc_ndcg_at_10_diff1": 0.5005185393647761, + "nauc_ndcg_at_10_max": 0.49409550921802203, + "nauc_ndcg_at_10_std": 0.2573950043468117, + "nauc_ndcg_at_1_diff1": 0.7959089560806106, + "nauc_ndcg_at_1_max": 0.5833537045527906, + "nauc_ndcg_at_1_std": 0.2247460991649088, + "nauc_ndcg_at_20_diff1": 0.4983481121295366, + "nauc_ndcg_at_20_max": 0.4957583602414652, + "nauc_ndcg_at_20_std": 0.2670278779770772, + "nauc_ndcg_at_3_diff1": 0.5237347007876767, + "nauc_ndcg_at_3_max": 0.49338833278907196, + "nauc_ndcg_at_3_std": 0.24110227461528355, + "nauc_ndcg_at_5_diff1": 0.5124601222416136, + "nauc_ndcg_at_5_max": 0.49338102495399494, + "nauc_ndcg_at_5_std": 0.24759886717439966, + "nauc_precision_at_1000_diff1": 0.13371828999244276, + "nauc_precision_at_1000_max": 0.38242817230874293, + "nauc_precision_at_1000_std": 0.4866512108820608, + "nauc_precision_at_100_diff1": 0.22052245553959812, + "nauc_precision_at_100_max": 0.39175435806910075, + "nauc_precision_at_100_std": 0.3771142121189264, + "nauc_precision_at_10_diff1": 0.32238099759258354, + "nauc_precision_at_10_max": 0.41500519303055844, + "nauc_precision_at_10_std": 0.26203664924573694, + "nauc_precision_at_1_diff1": 0.7959089560806106, + "nauc_precision_at_1_max": 0.5833537045527906, + "nauc_precision_at_1_std": 0.2247460991649088, + "nauc_precision_at_20_diff1": 0.28701307289428374, + "nauc_precision_at_20_max": 0.39944357620133486, + "nauc_precision_at_20_std": 0.28637330450968107, + "nauc_precision_at_3_diff1": 0.4162491563002872, + "nauc_precision_at_3_max": 0.45165029358949615, + "nauc_precision_at_3_std": 0.2431369436683493, + "nauc_precision_at_5_diff1": 0.37329607439227735, + "nauc_precision_at_5_max": 0.431329196973235, + "nauc_precision_at_5_std": 0.24569985906517514, + "nauc_recall_at_1000_diff1": 0.13371828999244287, + "nauc_recall_at_1000_max": 0.3824281723087443, + "nauc_recall_at_1000_std": 0.48665121088206087, + "nauc_recall_at_100_diff1": 0.22052245553959793, + "nauc_recall_at_100_max": 0.39175435806910053, + "nauc_recall_at_100_std": 0.3771142121189255, + "nauc_recall_at_10_diff1": 0.32238099759258254, + "nauc_recall_at_10_max": 0.4150051930305588, + "nauc_recall_at_10_std": 0.2620366492457372, + "nauc_recall_at_1_diff1": 0.7959089560806106, + "nauc_recall_at_1_max": 0.5833537045527906, + "nauc_recall_at_1_std": 0.2247460991649088, + "nauc_recall_at_20_diff1": 0.28701307289428274, + "nauc_recall_at_20_max": 0.3994435762013338, + "nauc_recall_at_20_std": 0.2863733045096808, + "nauc_recall_at_3_diff1": 0.41624915630028764, + "nauc_recall_at_3_max": 0.4516502935894968, + "nauc_recall_at_3_std": 0.2431369436683497, + "nauc_recall_at_5_diff1": 0.37329607439227647, + "nauc_recall_at_5_max": 0.4313291969732342, + "nauc_recall_at_5_std": 0.2456998590651744, + "ndcg_at_1": 0.68239, + "ndcg_at_10": 0.59451, + "ndcg_at_100": 0.63221, + "ndcg_at_1000": 0.64688, + "ndcg_at_20": 0.61134, + "ndcg_at_3": 0.54349, + "ndcg_at_5": 0.5697, + "precision_at_1": 0.68239, + "precision_at_10": 0.12539, + "precision_at_100": 0.01546, + "precision_at_1000": 0.00174, + "precision_at_20": 0.06812, + "precision_at_3": 0.34208, + "precision_at_5": 0.22596, + "recall_at_1": 0.3412, + "recall_at_10": 0.62695, + "recall_at_100": 0.77309, + "recall_at_1000": 0.87011, + "recall_at_20": 0.6812, + "recall_at_3": 0.51313, + "recall_at_5": 0.5649 + } + ], + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.56907, + "map_at_1": 0.32924, + "map_at_10": 0.4751, + "map_at_100": 0.48596, + "map_at_1000": 0.48675, + "map_at_20": 0.48142, + "map_at_3": 0.44324, + "map_at_5": 0.46202, + "mrr_at_1": 0.6582039162727886, + "mrr_at_10": 0.7325614931995739, + "mrr_at_100": 0.7363912796435997, + "mrr_at_1000": 0.7365216214733555, + "mrr_at_20": 0.735011291490657, + "mrr_at_3": 0.7171280666216512, + "mrr_at_5": 0.7271280666216491, + "nauc_map_at_1000_diff1": 0.41688853802189474, + "nauc_map_at_1000_max": 0.373382579309616, + "nauc_map_at_1000_std": 0.11882451687261855, + "nauc_map_at_100_diff1": 0.4167556624206158, + "nauc_map_at_100_max": 0.37322157861554656, + "nauc_map_at_100_std": 0.11849369950078638, + "nauc_map_at_10_diff1": 0.4192563184683788, + "nauc_map_at_10_max": 0.37218166021661986, + "nauc_map_at_10_std": 0.11165963760396123, + "nauc_map_at_1_diff1": 0.7582744508739551, + "nauc_map_at_1_max": 0.4728922037095045, + "nauc_map_at_1_std": 0.071023909474976, + "nauc_map_at_20_diff1": 0.41773049228261383, + "nauc_map_at_20_max": 0.3728651125597511, + "nauc_map_at_20_std": 0.11546516219738656, + "nauc_map_at_3_diff1": 0.43990332381393193, + "nauc_map_at_3_max": 0.3771572731854684, + "nauc_map_at_3_std": 0.10380181554429131, + "nauc_map_at_5_diff1": 0.4271936194233336, + "nauc_map_at_5_max": 0.37603547238332236, + "nauc_map_at_5_std": 0.10910086102850437, + "nauc_mrr_at_1000_diff1": 0.7465930642498361, + "nauc_mrr_at_1000_max": 0.49129937676197255, + "nauc_mrr_at_1000_std": 0.08879459347459807, + "nauc_mrr_at_100_diff1": 0.7465724983443507, + "nauc_mrr_at_100_max": 0.4913449382474845, + "nauc_mrr_at_100_std": 0.088871865850516, + "nauc_mrr_at_10_diff1": 0.7460101660533552, + "nauc_mrr_at_10_max": 0.49113565103792894, + "nauc_mrr_at_10_std": 0.08756833847815539, + "nauc_mrr_at_1_diff1": 0.7588291404831541, + "nauc_mrr_at_1_max": 0.4727752227472813, + "nauc_mrr_at_1_std": 0.07198517634791765, + "nauc_mrr_at_20_diff1": 0.7462400800015456, + "nauc_mrr_at_20_max": 0.4911675724289531, + "nauc_mrr_at_20_std": 0.08830808816293369, + "nauc_mrr_at_3_diff1": 0.7463166087254373, + "nauc_mrr_at_3_max": 0.4899367491846199, + "nauc_mrr_at_3_std": 0.08404979237119185, + "nauc_mrr_at_5_diff1": 0.7460990138441059, + "nauc_mrr_at_5_max": 0.4910213653465244, + "nauc_mrr_at_5_std": 0.08559755933007858, + "nauc_ndcg_at_1000_diff1": 0.4507609326184009, + "nauc_ndcg_at_1000_max": 0.3972460728985968, + "nauc_ndcg_at_1000_std": 0.14855567886235715, + "nauc_ndcg_at_100_diff1": 0.44617339411602486, + "nauc_ndcg_at_100_max": 0.3947680092007449, + "nauc_ndcg_at_100_std": 0.144275998945168, + "nauc_ndcg_at_10_diff1": 0.45840060377577196, + "nauc_ndcg_at_10_max": 0.39152456055658513, + "nauc_ndcg_at_10_std": 0.11575301935639057, + "nauc_ndcg_at_1_diff1": 0.7582744508739551, + "nauc_ndcg_at_1_max": 0.4728922037095045, + "nauc_ndcg_at_1_std": 0.071023909474976, + "nauc_ndcg_at_20_diff1": 0.4523346535223582, + "nauc_ndcg_at_20_max": 0.39224690249854993, + "nauc_ndcg_at_20_std": 0.12584079292768427, + "nauc_ndcg_at_3_diff1": 0.49543141865185014, + "nauc_ndcg_at_3_max": 0.4021504401764155, + "nauc_ndcg_at_3_std": 0.10301319921096169, + "nauc_ndcg_at_5_diff1": 0.47501531172507067, + "nauc_ndcg_at_5_max": 0.3988998172204565, + "nauc_ndcg_at_5_std": 0.10991519021424365, + "nauc_precision_at_1000_diff1": 0.011510731124192692, + "nauc_precision_at_1000_max": 0.2359197917285265, + "nauc_precision_at_1000_std": 0.35898993090133374, + "nauc_precision_at_100_diff1": 0.10819700477690608, + "nauc_precision_at_100_max": 0.26257900526895916, + "nauc_precision_at_100_std": 0.25839913764647304, + "nauc_precision_at_10_diff1": 0.2585720793609862, + "nauc_precision_at_10_max": 0.30528596354360116, + "nauc_precision_at_10_std": 0.13211209426079382, + "nauc_precision_at_1_diff1": 0.7582744508739551, + "nauc_precision_at_1_max": 0.4728922037095045, + "nauc_precision_at_1_std": 0.071023909474976, + "nauc_precision_at_20_diff1": 0.21286936748309704, + "nauc_precision_at_20_max": 0.2886443663784865, + "nauc_precision_at_20_std": 0.16030560770789795, + "nauc_precision_at_3_diff1": 0.3767274371572688, + "nauc_precision_at_3_max": 0.36123833536736266, + "nauc_precision_at_3_std": 0.11303437929682084, + "nauc_precision_at_5_diff1": 0.3241768117105102, + "nauc_precision_at_5_max": 0.3436804362651512, + "nauc_precision_at_5_std": 0.12310017843441869, + "nauc_recall_at_1000_diff1": 0.011510731124195383, + "nauc_recall_at_1000_max": 0.2359197917285284, + "nauc_recall_at_1000_std": 0.3589899309013362, + "nauc_recall_at_100_diff1": 0.10819700477690601, + "nauc_recall_at_100_max": 0.2625790052689587, + "nauc_recall_at_100_std": 0.2583991376464734, + "nauc_recall_at_10_diff1": 0.25857207936098575, + "nauc_recall_at_10_max": 0.3052859635436007, + "nauc_recall_at_10_std": 0.13211209426079396, + "nauc_recall_at_1_diff1": 0.7582744508739551, + "nauc_recall_at_1_max": 0.4728922037095045, + "nauc_recall_at_1_std": 0.071023909474976, + "nauc_recall_at_20_diff1": 0.2128693674830978, + "nauc_recall_at_20_max": 0.2886443663784866, + "nauc_recall_at_20_std": 0.16030560770789842, + "nauc_recall_at_3_diff1": 0.37672743715726864, + "nauc_recall_at_3_max": 0.36123833536736233, + "nauc_recall_at_3_std": 0.11303437929682086, + "nauc_recall_at_5_diff1": 0.32417681171051055, + "nauc_recall_at_5_max": 0.34368043626515116, + "nauc_recall_at_5_std": 0.12310017843441892, + "ndcg_at_1": 0.65847, + "ndcg_at_10": 0.56907, + "ndcg_at_100": 0.60958, + "ndcg_at_1000": 0.62509, + "ndcg_at_20": 0.58589, + "ndcg_at_3": 0.51945, + "ndcg_at_5": 0.54569, + "precision_at_1": 0.65847, + "precision_at_10": 0.11992, + "precision_at_100": 0.01517, + "precision_at_1000": 0.00172, + "precision_at_20": 0.06537, + "precision_at_3": 0.32595, + "precision_at_5": 0.21631, + "recall_at_1": 0.32924, + "recall_at_10": 0.59959, + "recall_at_100": 0.75868, + "recall_at_1000": 0.86117, + "recall_at_20": 0.65375, + "recall_at_3": 0.48893, + "recall_at_5": 0.54078 + } + ], + "train": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.58909, + "map_at_1": 0.33906, + "map_at_10": 0.49678, + "map_at_100": 0.50723, + "map_at_1000": 0.50798, + "map_at_20": 0.50288, + "map_at_3": 0.46389, + "map_at_5": 0.48341, + "mrr_at_1": 0.6782117647058824, + "mrr_at_10": 0.7482686601307064, + "mrr_at_100": 0.7518450894993166, + "mrr_at_1000": 0.7519729612189524, + "mrr_at_20": 0.7505932238242149, + "mrr_at_3": 0.7336058823529765, + "mrr_at_5": 0.7425564705882529, + "nauc_map_at_1000_diff1": 0.44743499615696297, + "nauc_map_at_1000_max": 0.45239181227114994, + "nauc_map_at_1000_std": 0.22742636228535132, + "nauc_map_at_100_diff1": 0.4472236397570774, + "nauc_map_at_100_max": 0.4521799780720236, + "nauc_map_at_100_std": 0.2270756642400961, + "nauc_map_at_10_diff1": 0.44829065475756386, + "nauc_map_at_10_max": 0.45076372662953634, + "nauc_map_at_10_std": 0.22153577163373878, + "nauc_map_at_1_diff1": 0.7822041605691044, + "nauc_map_at_1_max": 0.5450657119510068, + "nauc_map_at_1_std": 0.18439630688145278, + "nauc_map_at_20_diff1": 0.44750266558838064, + "nauc_map_at_20_max": 0.4515686143025165, + "nauc_map_at_20_std": 0.2243245143191772, + "nauc_map_at_3_diff1": 0.4606383901519939, + "nauc_map_at_3_max": 0.45105991724747607, + "nauc_map_at_3_std": 0.21350595260457833, + "nauc_map_at_5_diff1": 0.45281415815038795, + "nauc_map_at_5_max": 0.4512454899335895, + "nauc_map_at_5_std": 0.21798889741649727, + "nauc_mrr_at_1000_diff1": 0.770759828496424, + "nauc_mrr_at_1000_max": 0.5657189996121711, + "nauc_mrr_at_1000_std": 0.1989779156958595, + "nauc_mrr_at_100_diff1": 0.7707448700427082, + "nauc_mrr_at_100_max": 0.5657744431819042, + "nauc_mrr_at_100_std": 0.19905353839701623, + "nauc_mrr_at_10_diff1": 0.770260879569188, + "nauc_mrr_at_10_max": 0.5659139998332322, + "nauc_mrr_at_10_std": 0.19804608757916653, + "nauc_mrr_at_1_diff1": 0.7820075661846756, + "nauc_mrr_at_1_max": 0.5451459225159444, + "nauc_mrr_at_1_std": 0.18453812074661266, + "nauc_mrr_at_20_diff1": 0.7704334305690443, + "nauc_mrr_at_20_max": 0.5658280334165973, + "nauc_mrr_at_20_std": 0.19864777026039418, + "nauc_mrr_at_3_diff1": 0.7698228017186061, + "nauc_mrr_at_3_max": 0.564612154275569, + "nauc_mrr_at_3_std": 0.1954841675841319, + "nauc_mrr_at_5_diff1": 0.7703479031077959, + "nauc_mrr_at_5_max": 0.5658397572543362, + "nauc_mrr_at_5_std": 0.19666933076338794, + "nauc_ndcg_at_1000_diff1": 0.48665383603055595, + "nauc_ndcg_at_1000_max": 0.479455700187694, + "nauc_ndcg_at_1000_std": 0.25443150009019205, + "nauc_ndcg_at_100_diff1": 0.4811386881430355, + "nauc_ndcg_at_100_max": 0.47506280876837476, + "nauc_ndcg_at_100_std": 0.24878901813988347, + "nauc_ndcg_at_10_diff1": 0.4871732837843692, + "nauc_ndcg_at_10_max": 0.4708484056184829, + "nauc_ndcg_at_10_std": 0.22503644559942118, + "nauc_ndcg_at_1_diff1": 0.7822041605691044, + "nauc_ndcg_at_1_max": 0.5450657119510068, + "nauc_ndcg_at_1_std": 0.18439630688145278, + "nauc_ndcg_at_20_diff1": 0.48305692758000673, + "nauc_ndcg_at_20_max": 0.4718459016594572, + "nauc_ndcg_at_20_std": 0.23272184929693848, + "nauc_ndcg_at_3_diff1": 0.5126133435194796, + "nauc_ndcg_at_3_max": 0.47462756192716754, + "nauc_ndcg_at_3_std": 0.21248114846010765, + "nauc_ndcg_at_5_diff1": 0.4984463865042974, + "nauc_ndcg_at_5_max": 0.4731082543785899, + "nauc_ndcg_at_5_std": 0.21826956161827135, + "nauc_precision_at_1000_diff1": 0.10814305547069407, + "nauc_precision_at_1000_max": 0.3496202301344932, + "nauc_precision_at_1000_std": 0.4358083282381601, + "nauc_precision_at_100_diff1": 0.18187454615042375, + "nauc_precision_at_100_max": 0.3476552554489612, + "nauc_precision_at_100_std": 0.3255497131215264, + "nauc_precision_at_10_diff1": 0.3069730744925411, + "nauc_precision_at_10_max": 0.39422768288174365, + "nauc_precision_at_10_std": 0.2312707902251843, + "nauc_precision_at_1_diff1": 0.7822041605691044, + "nauc_precision_at_1_max": 0.5450657119510068, + "nauc_precision_at_1_std": 0.18439630688145278, + "nauc_precision_at_20_diff1": 0.2621742295102127, + "nauc_precision_at_20_max": 0.37306367306382154, + "nauc_precision_at_20_std": 0.2474205735325766, + "nauc_precision_at_3_diff1": 0.40269288827164945, + "nauc_precision_at_3_max": 0.4386755300991332, + "nauc_precision_at_3_std": 0.21876959129079618, + "nauc_precision_at_5_diff1": 0.3549116539864632, + "nauc_precision_at_5_max": 0.41641996967765416, + "nauc_precision_at_5_std": 0.22122638396793068, + "nauc_recall_at_1000_diff1": 0.10814305547069429, + "nauc_recall_at_1000_max": 0.34962023013449384, + "nauc_recall_at_1000_std": 0.4358083282381595, + "nauc_recall_at_100_diff1": 0.18187454615042362, + "nauc_recall_at_100_max": 0.34765525544896153, + "nauc_recall_at_100_std": 0.32554971312152575, + "nauc_recall_at_10_diff1": 0.30697307449253997, + "nauc_recall_at_10_max": 0.39422768288174237, + "nauc_recall_at_10_std": 0.2312707902251837, + "nauc_recall_at_1_diff1": 0.7822041605691044, + "nauc_recall_at_1_max": 0.5450657119510068, + "nauc_recall_at_1_std": 0.18439630688145278, + "nauc_recall_at_20_diff1": 0.26217422951021285, + "nauc_recall_at_20_max": 0.3730636730638219, + "nauc_recall_at_20_std": 0.2474205735325768, + "nauc_recall_at_3_diff1": 0.4026928882716497, + "nauc_recall_at_3_max": 0.43867553009913285, + "nauc_recall_at_3_std": 0.21876959129079596, + "nauc_recall_at_5_diff1": 0.3549116539864634, + "nauc_recall_at_5_max": 0.4164199696776544, + "nauc_recall_at_5_std": 0.22122638396793037, + "ndcg_at_1": 0.67812, + "ndcg_at_10": 0.58909, + "ndcg_at_100": 0.62745, + "ndcg_at_1000": 0.6424, + "ndcg_at_20": 0.60522, + "ndcg_at_3": 0.53866, + "ndcg_at_5": 0.56516, + "precision_at_1": 0.67812, + "precision_at_10": 0.12414, + "precision_at_100": 0.01541, + "precision_at_1000": 0.00174, + "precision_at_20": 0.06725, + "precision_at_3": 0.3388, + "precision_at_5": 0.22426, + "recall_at_1": 0.33906, + "recall_at_10": 0.6207, + "recall_at_100": 0.77057, + "recall_at_1000": 0.86931, + "recall_at_20": 0.67255, + "recall_at_3": 0.50821, + "recall_at_5": 0.56065 + } + ] + }, + "task_name": "HotpotQA" +} \ No newline at end of file diff --git a/results/bm25s/0_1_10/MSMARCO.json b/results/bm25s/0_1_10/MSMARCO.json new file mode 100644 index 0000000000..ea58f58d2a --- /dev/null +++ b/results/bm25s/0_1_10/MSMARCO.json @@ -0,0 +1,456 @@ +{ + "dataset_revision": "c5a29a104738b98a9e76336939199e264163d4a0", + "evaluation_time": 44062.399728536606, + "kg_co2_emissions": null, + "mteb_version": "1.12.22", + "scores": { + "dev": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.21888, + "map_at_1": 0.09265, + "map_at_10": 0.17088, + "map_at_100": 0.1824, + "map_at_1000": 0.18327, + "map_at_20": 0.17734, + "map_at_3": 0.14305, + "map_at_5": 0.15797, + "mrr_at_1": 0.09627507163323783, + "mrr_at_10": 0.17484740983308322, + "mrr_at_100": 0.1861654960150656, + "mrr_at_1000": 0.18699827276188322, + "mrr_at_20": 0.18122796369082717, + "mrr_at_3": 0.14687201528175814, + "mrr_at_5": 0.16216571155682985, + "nauc_map_at_1000_diff1": 0.25216511362095445, + "nauc_map_at_1000_max": 0.19724833585253204, + "nauc_map_at_1000_std": 0.002805957551139326, + "nauc_map_at_100_diff1": 0.25207675971842736, + "nauc_map_at_100_max": 0.1970833452570442, + "nauc_map_at_100_std": 0.0022285648207927013, + "nauc_map_at_10_diff1": 0.2527319498345974, + "nauc_map_at_10_max": 0.19409446686915263, + "nauc_map_at_10_std": -0.006927835582766535, + "nauc_map_at_1_diff1": 0.2852247046489951, + "nauc_map_at_1_max": 0.16673300711719163, + "nauc_map_at_1_std": -0.01632832575881201, + "nauc_map_at_20_diff1": 0.2516644591299103, + "nauc_map_at_20_max": 0.19614378506694785, + "nauc_map_at_20_std": -0.0024377908167233548, + "nauc_map_at_3_diff1": 0.25516173290519134, + "nauc_map_at_3_max": 0.18308350872265514, + "nauc_map_at_3_std": -0.012736297881117041, + "nauc_map_at_5_diff1": 0.2525278708380518, + "nauc_map_at_5_max": 0.19030787879208877, + "nauc_map_at_5_std": -0.009798722346281959, + "nauc_mrr_at_1000_diff1": 0.24598193580993793, + "nauc_mrr_at_1000_max": 0.19321975017889897, + "nauc_mrr_at_1000_std": 0.002681859480468282, + "nauc_mrr_at_100_diff1": 0.24588663279980147, + "nauc_mrr_at_100_max": 0.19308867046348271, + "nauc_mrr_at_100_std": 0.002226390513606012, + "nauc_mrr_at_10_diff1": 0.24636172438937787, + "nauc_mrr_at_10_max": 0.19003863842319835, + "nauc_mrr_at_10_std": -0.006704716431737518, + "nauc_mrr_at_1_diff1": 0.2726959478451965, + "nauc_mrr_at_1_max": 0.16023835077363383, + "nauc_mrr_at_1_std": -0.018363422643004078, + "nauc_mrr_at_20_diff1": 0.24550992686993994, + "nauc_mrr_at_20_max": 0.19240354552283323, + "nauc_mrr_at_20_std": -0.0017401481777834028, + "nauc_mrr_at_3_diff1": 0.24761350326949488, + "nauc_mrr_at_3_max": 0.17985413447421839, + "nauc_mrr_at_3_std": -0.014229656988713691, + "nauc_mrr_at_5_diff1": 0.24604204122692003, + "nauc_mrr_at_5_max": 0.18644301610497607, + "nauc_mrr_at_5_std": -0.00975745154886299, + "nauc_ndcg_at_1000_diff1": 0.2440413425418809, + "nauc_ndcg_at_1000_max": 0.22211500846332066, + "nauc_ndcg_at_1000_std": 0.057005780006344764, + "nauc_ndcg_at_100_diff1": 0.2427364923056901, + "nauc_ndcg_at_100_max": 0.21698822095873396, + "nauc_ndcg_at_100_std": 0.045210154807500014, + "nauc_ndcg_at_10_diff1": 0.24471072454210754, + "nauc_ndcg_at_10_max": 0.20589352666976743, + "nauc_ndcg_at_10_std": 0.00039774572313470963, + "nauc_ndcg_at_1_diff1": 0.2803303706174648, + "nauc_ndcg_at_1_max": 0.16309601793905523, + "nauc_ndcg_at_1_std": -0.01600688426553731, + "nauc_ndcg_at_20_diff1": 0.24120987057900803, + "nauc_ndcg_at_20_max": 0.2113786372037079, + "nauc_ndcg_at_20_std": 0.014474306293285163, + "nauc_ndcg_at_3_diff1": 0.2472401974583551, + "nauc_ndcg_at_3_max": 0.1862364458183931, + "nauc_ndcg_at_3_std": -0.0112794193697167, + "nauc_ndcg_at_5_diff1": 0.24349808340737958, + "nauc_ndcg_at_5_max": 0.19810241486101104, + "nauc_ndcg_at_5_std": -0.005854036836425033, + "nauc_precision_at_1000_diff1": 0.1234464902186415, + "nauc_precision_at_1000_max": 0.294149947465852, + "nauc_precision_at_1000_std": 0.37949979986991467, + "nauc_precision_at_100_diff1": 0.19127718983476652, + "nauc_precision_at_100_max": 0.2624030188899172, + "nauc_precision_at_100_std": 0.20321495957364233, + "nauc_precision_at_10_diff1": 0.2229543532920472, + "nauc_precision_at_10_max": 0.23210460893409574, + "nauc_precision_at_10_std": 0.017909089053392075, + "nauc_precision_at_1_diff1": 0.2803303706174648, + "nauc_precision_at_1_max": 0.16309601793905523, + "nauc_precision_at_1_std": -0.01600688426553731, + "nauc_precision_at_20_diff1": 0.21004431509807897, + "nauc_precision_at_20_max": 0.2456128195755701, + "nauc_precision_at_20_std": 0.0585379999689295, + "nauc_precision_at_3_diff1": 0.2296999902155528, + "nauc_precision_at_3_max": 0.19406885011447372, + "nauc_precision_at_3_std": -0.009198489600863398, + "nauc_precision_at_5_diff1": 0.22275045705092902, + "nauc_precision_at_5_max": 0.21551543391915523, + "nauc_precision_at_5_std": 0.002432706231372534, + "nauc_recall_at_1000_diff1": 0.2175144188556066, + "nauc_recall_at_1000_max": 0.4016847988230547, + "nauc_recall_at_1000_std": 0.4974981503996402, + "nauc_recall_at_100_diff1": 0.22016420872254172, + "nauc_recall_at_100_max": 0.27687201784492027, + "nauc_recall_at_100_std": 0.20769592728096375, + "nauc_recall_at_10_diff1": 0.23003310505861552, + "nauc_recall_at_10_max": 0.23094451958987575, + "nauc_recall_at_10_std": 0.015036561852436051, + "nauc_recall_at_1_diff1": 0.2852247046489951, + "nauc_recall_at_1_max": 0.16673300711719163, + "nauc_recall_at_1_std": -0.01632832575881201, + "nauc_recall_at_20_diff1": 0.21929172040246211, + "nauc_recall_at_20_max": 0.24440853312371233, + "nauc_recall_at_20_std": 0.0545550012236764, + "nauc_recall_at_3_diff1": 0.23107466999039464, + "nauc_recall_at_3_max": 0.19407872498072784, + "nauc_recall_at_3_std": -0.0076449068060041965, + "nauc_recall_at_5_diff1": 0.22596784768206069, + "nauc_recall_at_5_max": 0.21560515983465459, + "nauc_recall_at_5_std": 0.002165720839477495, + "ndcg_at_1": 0.09542, + "ndcg_at_10": 0.21888, + "ndcg_at_100": 0.27901, + "ndcg_at_1000": 0.30371, + "ndcg_at_20": 0.24237, + "ndcg_at_3": 0.16114, + "ndcg_at_5": 0.18796, + "precision_at_1": 0.09542, + "precision_at_10": 0.03848, + "precision_at_100": 0.00691, + "precision_at_1000": 0.0009, + "precision_at_20": 0.02406, + "precision_at_3": 0.07254, + "precision_at_5": 0.05702, + "recall_at_1": 0.09265, + "recall_at_10": 0.36927, + "recall_at_100": 0.657, + "recall_at_1000": 0.85209, + "recall_at_20": 0.46133, + "recall_at_3": 0.21003, + "recall_at_5": 0.27453 + } + ], + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.46311, + "map_at_1": 0.01787, + "map_at_10": 0.10212, + "map_at_100": 0.26346, + "map_at_1000": 0.33555, + "map_at_20": 0.14666, + "map_at_3": 0.04241, + "map_at_5": 0.06737, + "mrr_at_1": 0.6744186046511628, + "mrr_at_10": 0.7948320413436692, + "mrr_at_100": 0.7953376025165712, + "mrr_at_1000": 0.7953376025165712, + "mrr_at_20": 0.7948320413436692, + "mrr_at_3": 0.7829457364341085, + "mrr_at_5": 0.7922480620155038, + "nauc_map_at_1000_diff1": -0.23134653464282284, + "nauc_map_at_1000_max": 0.3813551810429688, + "nauc_map_at_1000_std": 0.39004992709546915, + "nauc_map_at_100_diff1": -0.14246130612522284, + "nauc_map_at_100_max": 0.24016572639718123, + "nauc_map_at_100_std": 0.18788938015820084, + "nauc_map_at_10_diff1": -0.03281248557606052, + "nauc_map_at_10_max": 0.046412353997225334, + "nauc_map_at_10_std": -0.09626113367061678, + "nauc_map_at_1_diff1": -0.028016447727928846, + "nauc_map_at_1_max": -0.12471032469174186, + "nauc_map_at_1_std": -0.255966395242003, + "nauc_map_at_20_diff1": -0.02646067727529821, + "nauc_map_at_20_max": 0.13626562077919988, + "nauc_map_at_20_std": 0.004023965914913995, + "nauc_map_at_3_diff1": -0.05354636047295973, + "nauc_map_at_3_max": -0.06461765592889668, + "nauc_map_at_3_std": -0.20544844658092715, + "nauc_map_at_5_diff1": -0.11621664116178215, + "nauc_map_at_5_max": -0.11129813364369077, + "nauc_map_at_5_std": -0.23339768093599814, + "nauc_mrr_at_1000_diff1": -0.008372625354656217, + "nauc_mrr_at_1000_max": 0.5500669345909858, + "nauc_mrr_at_1000_std": 0.38376310555410253, + "nauc_mrr_at_100_diff1": -0.008372625354656217, + "nauc_mrr_at_100_max": 0.5500669345909858, + "nauc_mrr_at_100_std": 0.38376310555410253, + "nauc_mrr_at_10_diff1": -0.008859301390673539, + "nauc_mrr_at_10_max": 0.5488412945666411, + "nauc_mrr_at_10_std": 0.3810632459842584, + "nauc_mrr_at_1_diff1": 0.04522083048274949, + "nauc_mrr_at_1_max": 0.5919782170244965, + "nauc_mrr_at_1_std": 0.41022576265569594, + "nauc_mrr_at_20_diff1": -0.008859301390673539, + "nauc_mrr_at_20_max": 0.5488412945666411, + "nauc_mrr_at_20_std": 0.3810632459842584, + "nauc_mrr_at_3_diff1": -0.02882041518897772, + "nauc_mrr_at_3_max": 0.5541068700650966, + "nauc_mrr_at_3_std": 0.407831271228486, + "nauc_mrr_at_5_diff1": -0.02488030586475158, + "nauc_mrr_at_5_max": 0.5426838099463793, + "nauc_mrr_at_5_std": 0.3807383694376633, + "nauc_ndcg_at_1000_diff1": -0.12670502733199493, + "nauc_ndcg_at_1000_max": 0.49070246457266686, + "nauc_ndcg_at_1000_std": 0.5154705921700614, + "nauc_ndcg_at_100_diff1": -0.12756073101804946, + "nauc_ndcg_at_100_max": 0.3351267684891069, + "nauc_ndcg_at_100_std": 0.32499117788461346, + "nauc_ndcg_at_10_diff1": -0.086999176726165, + "nauc_ndcg_at_10_max": 0.3114592848070508, + "nauc_ndcg_at_10_std": 0.22593373077953438, + "nauc_ndcg_at_1_diff1": 0.03374349568324434, + "nauc_ndcg_at_1_max": 0.364666514784135, + "nauc_ndcg_at_1_std": 0.2618675679740419, + "nauc_ndcg_at_20_diff1": -0.11045332928595163, + "nauc_ndcg_at_20_max": 0.2951686596024688, + "nauc_ndcg_at_20_std": 0.25030662054358904, + "nauc_ndcg_at_3_diff1": -0.029411833896165727, + "nauc_ndcg_at_3_max": 0.4011417865337722, + "nauc_ndcg_at_3_std": 0.2732892152067753, + "nauc_ndcg_at_5_diff1": -0.10059531615113682, + "nauc_ndcg_at_5_max": 0.2956245885114656, + "nauc_ndcg_at_5_std": 0.17640013374582325, + "nauc_precision_at_1000_diff1": -0.4168854639613429, + "nauc_precision_at_1000_max": 0.28505433547327313, + "nauc_precision_at_1000_std": 0.4631399357779993, + "nauc_precision_at_100_diff1": -0.3887548115718526, + "nauc_precision_at_100_max": 0.2897996828356092, + "nauc_precision_at_100_std": 0.45220460895537035, + "nauc_precision_at_10_diff1": -0.13950249715389196, + "nauc_precision_at_10_max": 0.34655069497607305, + "nauc_precision_at_10_std": 0.3107429626927814, + "nauc_precision_at_1_diff1": 0.04522083048274949, + "nauc_precision_at_1_max": 0.5919782170244965, + "nauc_precision_at_1_std": 0.41022576265569594, + "nauc_precision_at_20_diff1": -0.1751350279289043, + "nauc_precision_at_20_max": 0.3377624493087112, + "nauc_precision_at_20_std": 0.36534485406037587, + "nauc_precision_at_3_diff1": -0.1615658543390671, + "nauc_precision_at_3_max": 0.424849482505466, + "nauc_precision_at_3_std": 0.33264913967310694, + "nauc_precision_at_5_diff1": -0.1935395719764801, + "nauc_precision_at_5_max": 0.2729643195101289, + "nauc_precision_at_5_std": 0.18099381189280336, + "nauc_recall_at_1000_diff1": -0.08462585475090383, + "nauc_recall_at_1000_max": 0.4734805042729931, + "nauc_recall_at_1000_std": 0.5622297199417984, + "nauc_recall_at_100_diff1": -0.022211206601956874, + "nauc_recall_at_100_max": 0.15902258369620412, + "nauc_recall_at_100_std": 0.12789393853139988, + "nauc_recall_at_10_diff1": -0.01739128664637983, + "nauc_recall_at_10_max": -0.026917060905661594, + "nauc_recall_at_10_std": -0.1608734398024053, + "nauc_recall_at_1_diff1": -0.028016447727928846, + "nauc_recall_at_1_max": -0.12471032469174186, + "nauc_recall_at_1_std": -0.255966395242003, + "nauc_recall_at_20_diff1": 0.027712936318559492, + "nauc_recall_at_20_max": 0.060093613489547844, + "nauc_recall_at_20_std": -0.0653159103063631, + "nauc_recall_at_3_diff1": -0.052282220284469785, + "nauc_recall_at_3_max": -0.10554057438956874, + "nauc_recall_at_3_std": -0.23398163656607432, + "nauc_recall_at_5_diff1": -0.13160137881249123, + "nauc_recall_at_5_max": -0.19201160892914745, + "nauc_recall_at_5_std": -0.307053531882817, + "ndcg_at_1": 0.49225, + "ndcg_at_10": 0.46311, + "ndcg_at_100": 0.4644, + "ndcg_at_1000": 0.57601, + "ndcg_at_20": 0.45122, + "ndcg_at_3": 0.47209, + "ndcg_at_5": 0.46801, + "precision_at_1": 0.67442, + "precision_at_10": 0.57907, + "precision_at_100": 0.29651, + "precision_at_1000": 0.06363, + "precision_at_20": 0.50349, + "precision_at_3": 0.65116, + "precision_at_5": 0.6186, + "recall_at_1": 0.01787, + "recall_at_10": 0.12354, + "recall_at_100": 0.42363, + "recall_at_1000": 0.72438, + "recall_at_20": 0.18853, + "recall_at_3": 0.04598, + "recall_at_5": 0.07764 + } + ], + "train": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.19915, + "map_at_1": 0.08536, + "map_at_10": 0.15487, + "map_at_100": 0.16661, + "map_at_1000": 0.16757, + "map_at_20": 0.1614, + "map_at_3": 0.12808, + "map_at_5": 0.14245, + "mrr_at_1": 0.08867079307828583, + "mrr_at_10": 0.15879720116259635, + "mrr_at_100": 0.17034905781931753, + "mrr_at_1000": 0.17126189497855665, + "mrr_at_20": 0.16524399915155058, + "mrr_at_3": 0.1320607469295864, + "mrr_at_5": 0.14644688520871738, + "nauc_map_at_1000_diff1": 0.2323520494884941, + "nauc_map_at_1000_max": 0.16704769618039153, + "nauc_map_at_1000_std": -0.010297143707060967, + "nauc_map_at_100_diff1": 0.23229954131210626, + "nauc_map_at_100_max": 0.16690820706298332, + "nauc_map_at_100_std": -0.010791475140908082, + "nauc_map_at_10_diff1": 0.23359735328183773, + "nauc_map_at_10_max": 0.16414886928244288, + "nauc_map_at_10_std": -0.02051251917376209, + "nauc_map_at_1_diff1": 0.2775320565646951, + "nauc_map_at_1_max": 0.146151948372954, + "nauc_map_at_1_std": -0.032704279084440485, + "nauc_map_at_20_diff1": 0.2325621828427942, + "nauc_map_at_20_max": 0.16566953495764872, + "nauc_map_at_20_std": -0.016029879039150687, + "nauc_map_at_3_diff1": 0.2407008941598721, + "nauc_map_at_3_max": 0.15546685768403146, + "nauc_map_at_3_std": -0.027785479469324383, + "nauc_map_at_5_diff1": 0.23683385774318672, + "nauc_map_at_5_max": 0.160491384153811, + "nauc_map_at_5_std": -0.025068728442654935, + "nauc_mrr_at_1000_diff1": 0.22849199285833174, + "nauc_mrr_at_1000_max": 0.165026491036786, + "nauc_mrr_at_1000_std": -0.01020110524624802, + "nauc_mrr_at_100_diff1": 0.2284290114754509, + "nauc_mrr_at_100_max": 0.16491330614993385, + "nauc_mrr_at_100_std": -0.010603768351103209, + "nauc_mrr_at_10_diff1": 0.2296069359515518, + "nauc_mrr_at_10_max": 0.16231194892758224, + "nauc_mrr_at_10_std": -0.019899356241393173, + "nauc_mrr_at_1_diff1": 0.26894072502899796, + "nauc_mrr_at_1_max": 0.14533497364277287, + "nauc_mrr_at_1_std": -0.03172007907665317, + "nauc_mrr_at_20_diff1": 0.2286603251955594, + "nauc_mrr_at_20_max": 0.16377314770151483, + "nauc_mrr_at_20_std": -0.01552963076855472, + "nauc_mrr_at_3_diff1": 0.23616488611430617, + "nauc_mrr_at_3_max": 0.15382260618033472, + "nauc_mrr_at_3_std": -0.027387062252932498, + "nauc_mrr_at_5_diff1": 0.23276255333802726, + "nauc_mrr_at_5_max": 0.15890658535120858, + "nauc_mrr_at_5_std": -0.024452627301040166, + "nauc_ndcg_at_1000_diff1": 0.216091234251362, + "nauc_ndcg_at_1000_max": 0.18968642372801678, + "nauc_ndcg_at_1000_std": 0.04693129859769538, + "nauc_ndcg_at_100_diff1": 0.2151792811185184, + "nauc_ndcg_at_100_max": 0.18571105334882562, + "nauc_ndcg_at_100_std": 0.03744438643886652, + "nauc_ndcg_at_10_diff1": 0.21983590052123078, + "nauc_ndcg_at_10_max": 0.17232588489244136, + "nauc_ndcg_at_10_std": -0.012375684021163066, + "nauc_ndcg_at_1_diff1": 0.2742712298113122, + "nauc_ndcg_at_1_max": 0.1448372600831636, + "nauc_ndcg_at_1_std": -0.03256798931077233, + "nauc_ndcg_at_20_diff1": 0.2168704275923046, + "nauc_ndcg_at_20_max": 0.17699650265859335, + "nauc_ndcg_at_20_std": 0.0017935973320235062, + "nauc_ndcg_at_3_diff1": 0.23143279794243415, + "nauc_ndcg_at_3_max": 0.15707556729170236, + "nauc_ndcg_at_3_std": -0.02636397144161579, + "nauc_ndcg_at_5_diff1": 0.22595477163341557, + "nauc_ndcg_at_5_max": 0.1649885334263227, + "nauc_ndcg_at_5_std": -0.021963511919441694, + "nauc_precision_at_1000_diff1": 0.08991284357381017, + "nauc_precision_at_1000_max": 0.25635279756846796, + "nauc_precision_at_1000_std": 0.3580997187246212, + "nauc_precision_at_100_diff1": 0.15558247057242316, + "nauc_precision_at_100_max": 0.23038572533805157, + "nauc_precision_at_100_std": 0.20079369051603713, + "nauc_precision_at_10_diff1": 0.19056069235152642, + "nauc_precision_at_10_max": 0.1892189017031358, + "nauc_precision_at_10_std": 0.005872713476482668, + "nauc_precision_at_1_diff1": 0.2742712298113122, + "nauc_precision_at_1_max": 0.1448372600831636, + "nauc_precision_at_1_std": -0.03256798931077233, + "nauc_precision_at_20_diff1": 0.18053716203226086, + "nauc_precision_at_20_max": 0.19963250806600613, + "nauc_precision_at_20_std": 0.04476100573007537, + "nauc_precision_at_3_diff1": 0.21054939050461516, + "nauc_precision_at_3_max": 0.16072405938409517, + "nauc_precision_at_3_std": -0.0233002806069913, + "nauc_precision_at_5_diff1": 0.2029224464162659, + "nauc_precision_at_5_max": 0.17490157142385568, + "nauc_precision_at_5_std": -0.015153445415260307, + "nauc_recall_at_1000_diff1": 0.14296685264235062, + "nauc_recall_at_1000_max": 0.3534408355739073, + "nauc_recall_at_1000_std": 0.4841812042788738, + "nauc_recall_at_100_diff1": 0.17180282864059435, + "nauc_recall_at_100_max": 0.2447086970308505, + "nauc_recall_at_100_std": 0.21028517198377203, + "nauc_recall_at_10_diff1": 0.1924460483146572, + "nauc_recall_at_10_max": 0.18871740786365232, + "nauc_recall_at_10_std": 0.004845564134816213, + "nauc_recall_at_1_diff1": 0.2775320565646951, + "nauc_recall_at_1_max": 0.146151948372954, + "nauc_recall_at_1_std": -0.032704279084440485, + "nauc_recall_at_20_diff1": 0.18474975006421696, + "nauc_recall_at_20_max": 0.2009842900227081, + "nauc_recall_at_20_std": 0.0437913346884448, + "nauc_recall_at_3_diff1": 0.21231965641002137, + "nauc_recall_at_3_max": 0.16114121439462975, + "nauc_recall_at_3_std": -0.023319766717538763, + "nauc_recall_at_5_diff1": 0.20385506589939878, + "nauc_recall_at_5_max": 0.17413493458216092, + "nauc_recall_at_5_std": -0.015549437133338069, + "ndcg_at_1": 0.08811, + "ndcg_at_10": 0.19915, + "ndcg_at_100": 0.26086, + "ndcg_at_1000": 0.28785, + "ndcg_at_20": 0.22268, + "ndcg_at_3": 0.1434, + "ndcg_at_5": 0.16925, + "precision_at_1": 0.08811, + "precision_at_10": 0.03521, + "precision_at_100": 0.00666, + "precision_at_1000": 0.0009, + "precision_at_20": 0.02244, + "precision_at_3": 0.06371, + "precision_at_5": 0.05125, + "recall_at_1": 0.08536, + "recall_at_10": 0.33834, + "recall_at_100": 0.63537, + "recall_at_1000": 0.84909, + "recall_at_20": 0.43032, + "recall_at_3": 0.18446, + "recall_at_5": 0.24682 + } + ] + }, + "task_name": "MSMARCO" +} \ No newline at end of file diff --git a/results/bm25s/0_1_10/NFCorpus.json b/results/bm25s/0_1_10/NFCorpus.json new file mode 100644 index 0000000000..d2a23baf2f --- /dev/null +++ b/results/bm25s/0_1_10/NFCorpus.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ec0fa4fe99da2ff19ca1214b7966684033a58814", + "evaluation_time": 1.505730390548706, + "kg_co2_emissions": null, + "mteb_version": "1.12.22", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.32082, + "map_at_1": 0.05628, + "map_at_10": 0.11857, + "map_at_100": 0.14306, + "map_at_1000": 0.15017, + "map_at_20": 0.12949, + "map_at_3": 0.09345, + "map_at_5": 0.10424, + "mrr_at_1": 0.4458204334365325, + "mrr_at_10": 0.5291992235490686, + "mrr_at_100": 0.5328396005783896, + "mrr_at_1000": 0.5330333502604587, + "mrr_at_20": 0.5309545717440454, + "mrr_at_3": 0.5113519091847266, + "mrr_at_5": 0.5229618163054696, + "nauc_map_at_1000_diff1": 0.26918348098088474, + "nauc_map_at_1000_max": 0.35250262456844433, + "nauc_map_at_1000_std": 0.04649275803304677, + "nauc_map_at_100_diff1": 0.2844610717818622, + "nauc_map_at_100_max": 0.35724171816695854, + "nauc_map_at_100_std": 0.033648402491604675, + "nauc_map_at_10_diff1": 0.36081687164599285, + "nauc_map_at_10_max": 0.3667657239646644, + "nauc_map_at_10_std": -0.03229932625541164, + "nauc_map_at_1_diff1": 0.5627326977791751, + "nauc_map_at_1_max": 0.35586667523647353, + "nauc_map_at_1_std": -0.08650129005765297, + "nauc_map_at_20_diff1": 0.31625167761149636, + "nauc_map_at_20_max": 0.3525543173962236, + "nauc_map_at_20_std": -0.012658733039342226, + "nauc_map_at_3_diff1": 0.44354489651709644, + "nauc_map_at_3_max": 0.3728580261351689, + "nauc_map_at_3_std": -0.07182246767848623, + "nauc_map_at_5_diff1": 0.4054779536858048, + "nauc_map_at_5_max": 0.3771233860215608, + "nauc_map_at_5_std": -0.0478039883499618, + "nauc_mrr_at_1000_diff1": 0.26481405467021124, + "nauc_mrr_at_1000_max": 0.38948776191324486, + "nauc_mrr_at_1000_std": 0.07368994398426237, + "nauc_mrr_at_100_diff1": 0.2648582111805387, + "nauc_mrr_at_100_max": 0.38948840666691625, + "nauc_mrr_at_100_std": 0.07346852439718575, + "nauc_mrr_at_10_diff1": 0.26709842606791906, + "nauc_mrr_at_10_max": 0.3893564355732135, + "nauc_mrr_at_10_std": 0.06928928650137192, + "nauc_mrr_at_1_diff1": 0.2851846137646564, + "nauc_mrr_at_1_max": 0.33868797400680184, + "nauc_mrr_at_1_std": 0.010234032481511224, + "nauc_mrr_at_20_diff1": 0.26571417377801404, + "nauc_mrr_at_20_max": 0.38939310690392753, + "nauc_mrr_at_20_std": 0.0706806142294648, + "nauc_mrr_at_3_diff1": 0.25904889698082, + "nauc_mrr_at_3_max": 0.36860294661122706, + "nauc_mrr_at_3_std": 0.044839843391520404, + "nauc_mrr_at_5_diff1": 0.26960589154686077, + "nauc_mrr_at_5_max": 0.3881555082568393, + "nauc_mrr_at_5_std": 0.0635528387864065, + "nauc_ndcg_at_1000_diff1": 0.20725410116446266, + "nauc_ndcg_at_1000_max": 0.44937448679252595, + "nauc_ndcg_at_1000_std": 0.23065083861050717, + "nauc_ndcg_at_100_diff1": 0.21032951524940585, + "nauc_ndcg_at_100_max": 0.40801920701805033, + "nauc_ndcg_at_100_std": 0.1804573250325198, + "nauc_ndcg_at_10_diff1": 0.11096186553902558, + "nauc_ndcg_at_10_max": 0.2921276203541946, + "nauc_ndcg_at_10_std": 0.06593893621520551, + "nauc_ndcg_at_1_diff1": 0.30441868977288594, + "nauc_ndcg_at_1_max": 0.34064225025753125, + "nauc_ndcg_at_1_std": 0.008606763912151906, + "nauc_ndcg_at_20_diff1": 0.11790509594205417, + "nauc_ndcg_at_20_max": 0.29267475836328677, + "nauc_ndcg_at_20_std": 0.0879483645580819, + "nauc_ndcg_at_3_diff1": 0.14333521518237052, + "nauc_ndcg_at_3_max": 0.30753554508117714, + "nauc_ndcg_at_3_std": 0.029252109106690828, + "nauc_ndcg_at_5_diff1": 0.12583578294712258, + "nauc_ndcg_at_5_max": 0.3114217418709933, + "nauc_ndcg_at_5_std": 0.06054029842243836, + "nauc_precision_at_1000_diff1": -0.3257249041220851, + "nauc_precision_at_1000_max": -0.18044815224301092, + "nauc_precision_at_1000_std": 0.09340990941757926, + "nauc_precision_at_100_diff1": -0.2767002876551104, + "nauc_precision_at_100_max": -0.05247057470593053, + "nauc_precision_at_100_std": 0.2061539452437707, + "nauc_precision_at_10_diff1": -0.14955751515979857, + "nauc_precision_at_10_max": 0.12035424980932316, + "nauc_precision_at_10_std": 0.1194530222127563, + "nauc_precision_at_1_diff1": 0.3018740755481865, + "nauc_precision_at_1_max": 0.34416526405214426, + "nauc_precision_at_1_std": 0.01955191479108776, + "nauc_precision_at_20_diff1": -0.22421135707777895, + "nauc_precision_at_20_max": 0.04080525536528606, + "nauc_precision_at_20_std": 0.1464153503298407, + "nauc_precision_at_3_diff1": 0.003971419213691873, + "nauc_precision_at_3_max": 0.24886157646239254, + "nauc_precision_at_3_std": 0.06408623185207662, + "nauc_precision_at_5_diff1": -0.06381667669350159, + "nauc_precision_at_5_max": 0.21113495960173914, + "nauc_precision_at_5_std": 0.11230866090510294, + "nauc_recall_at_1000_diff1": 0.0756426528289946, + "nauc_recall_at_1000_max": 0.36069204543106304, + "nauc_recall_at_1000_std": 0.31806303246793655, + "nauc_recall_at_100_diff1": 0.20595566419978653, + "nauc_recall_at_100_max": 0.4419320428250885, + "nauc_recall_at_100_std": 0.2869120882408191, + "nauc_recall_at_10_diff1": 0.33138244816570206, + "nauc_recall_at_10_max": 0.4013218163043384, + "nauc_recall_at_10_std": 0.03117399554240831, + "nauc_recall_at_1_diff1": 0.5627326977791751, + "nauc_recall_at_1_max": 0.35586667523647353, + "nauc_recall_at_1_std": -0.08650129005765297, + "nauc_recall_at_20_diff1": 0.24113110007544242, + "nauc_recall_at_20_max": 0.3667516792470692, + "nauc_recall_at_20_std": 0.08852420576406517, + "nauc_recall_at_3_diff1": 0.4244171130872568, + "nauc_recall_at_3_max": 0.37635599727909247, + "nauc_recall_at_3_std": -0.057936548839647786, + "nauc_recall_at_5_diff1": 0.3714267043339234, + "nauc_recall_at_5_max": 0.3915011480327067, + "nauc_recall_at_5_std": -0.010054484415243814, + "ndcg_at_1": 0.42879, + "ndcg_at_10": 0.32082, + "ndcg_at_100": 0.27318, + "ndcg_at_1000": 0.32252, + "ndcg_at_20": 0.29417, + "ndcg_at_3": 0.39021, + "ndcg_at_5": 0.35562, + "precision_at_1": 0.43963, + "precision_at_10": 0.23158, + "precision_at_100": 0.06554, + "precision_at_1000": 0.01462, + "precision_at_20": 0.1661, + "precision_at_3": 0.36739, + "precision_at_5": 0.30031, + "recall_at_1": 0.05628, + "recall_at_10": 0.14707, + "recall_at_100": 0.24656, + "recall_at_1000": 0.4288, + "recall_at_20": 0.17655, + "recall_at_3": 0.10287, + "recall_at_5": 0.12078 + } + ] + }, + "task_name": "NFCorpus" +} \ No newline at end of file diff --git a/results/bm25s/0_1_10/NQ.json b/results/bm25s/0_1_10/NQ.json new file mode 100644 index 0000000000..07674e5d1d --- /dev/null +++ b/results/bm25s/0_1_10/NQ.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "b774495ed302d8c44a3a7ea25c90dbce03968f31", + "evaluation_time": 334.9865415096283, + "kg_co2_emissions": null, + "mteb_version": "1.12.22", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.285, + "map_at_1": 0.13526, + "map_at_10": 0.22702, + "map_at_100": 0.24099, + "map_at_1000": 0.24177, + "map_at_20": 0.23531, + "map_at_3": 0.19358, + "map_at_5": 0.2124, + "mrr_at_1": 0.15063731170336037, + "mrr_at_10": 0.24570308907649643, + "mrr_at_100": 0.2576914413729926, + "mrr_at_1000": 0.25830821416488237, + "mrr_at_20": 0.2528440845765534, + "mrr_at_3": 0.21325801467748184, + "mrr_at_5": 0.23174005407493317, + "nauc_map_at_1000_diff1": 0.2701761335203729, + "nauc_map_at_1000_max": 0.27135476699561306, + "nauc_map_at_1000_std": 0.10507205608933907, + "nauc_map_at_100_diff1": 0.27018469143439844, + "nauc_map_at_100_max": 0.27145752236819104, + "nauc_map_at_100_std": 0.10491414333927786, + "nauc_map_at_10_diff1": 0.2694120938725285, + "nauc_map_at_10_max": 0.26636704756603197, + "nauc_map_at_10_std": 0.09642226022974569, + "nauc_map_at_1_diff1": 0.31563754317382775, + "nauc_map_at_1_max": 0.22668271138365942, + "nauc_map_at_1_std": 0.055208961001451236, + "nauc_map_at_20_diff1": 0.2697638597826058, + "nauc_map_at_20_max": 0.2701278500903498, + "nauc_map_at_20_std": 0.10182784252732181, + "nauc_map_at_3_diff1": 0.2745037638194487, + "nauc_map_at_3_max": 0.24534534468503275, + "nauc_map_at_3_std": 0.08107887207044802, + "nauc_map_at_5_diff1": 0.26789591291178105, + "nauc_map_at_5_max": 0.2556447641999303, + "nauc_map_at_5_std": 0.08702548268102987, + "nauc_mrr_at_1000_diff1": 0.26924985376991645, + "nauc_mrr_at_1000_max": 0.26725004710524186, + "nauc_mrr_at_1000_std": 0.11080628756217101, + "nauc_mrr_at_100_diff1": 0.2692430083702952, + "nauc_mrr_at_100_max": 0.2672973477135129, + "nauc_mrr_at_100_std": 0.11074148187687624, + "nauc_mrr_at_10_diff1": 0.2678615552132973, + "nauc_mrr_at_10_max": 0.2640632811823152, + "nauc_mrr_at_10_std": 0.10522160429115344, + "nauc_mrr_at_1_diff1": 0.31185023699916714, + "nauc_mrr_at_1_max": 0.23242613899572206, + "nauc_mrr_at_1_std": 0.06860322319576374, + "nauc_mrr_at_20_diff1": 0.2686350159055199, + "nauc_mrr_at_20_max": 0.2663290702413082, + "nauc_mrr_at_20_std": 0.10867736444423304, + "nauc_mrr_at_3_diff1": 0.2708890172946183, + "nauc_mrr_at_3_max": 0.24613146608397413, + "nauc_mrr_at_3_std": 0.09149660758876045, + "nauc_mrr_at_5_diff1": 0.2650389797909921, + "nauc_mrr_at_5_max": 0.2548206516935941, + "nauc_mrr_at_5_std": 0.0976987729754947, + "nauc_ndcg_at_1000_diff1": 0.2640249927757632, + "nauc_ndcg_at_1000_max": 0.3046531307537997, + "nauc_ndcg_at_1000_std": 0.15497771135404423, + "nauc_ndcg_at_100_diff1": 0.26394666335866207, + "nauc_ndcg_at_100_max": 0.3071146206682209, + "nauc_ndcg_at_100_std": 0.15288318314714644, + "nauc_ndcg_at_10_diff1": 0.2582835439355767, + "nauc_ndcg_at_10_max": 0.28659538803302, + "nauc_ndcg_at_10_std": 0.11731061116985474, + "nauc_ndcg_at_1_diff1": 0.3101579718783373, + "nauc_ndcg_at_1_max": 0.232794151385547, + "nauc_ndcg_at_1_std": 0.06835199024544518, + "nauc_ndcg_at_20_diff1": 0.259941739452006, + "nauc_ndcg_at_20_max": 0.2979190428069538, + "nauc_ndcg_at_20_std": 0.13386810790412995, + "nauc_ndcg_at_3_diff1": 0.26349825000747845, + "nauc_ndcg_at_3_max": 0.24830871556872894, + "nauc_ndcg_at_3_std": 0.08832343198761994, + "nauc_ndcg_at_5_diff1": 0.2534831410139559, + "nauc_ndcg_at_5_max": 0.26384823426934867, + "nauc_ndcg_at_5_std": 0.09798689871511189, + "nauc_precision_at_1000_diff1": 0.04095224785215105, + "nauc_precision_at_1000_max": 0.20350150833232877, + "nauc_precision_at_1000_std": 0.2626397010457217, + "nauc_precision_at_100_diff1": 0.1315517750352698, + "nauc_precision_at_100_max": 0.3047151490336106, + "nauc_precision_at_100_std": 0.2723195942784396, + "nauc_precision_at_10_diff1": 0.20108744477849627, + "nauc_precision_at_10_max": 0.32861108719597815, + "nauc_precision_at_10_std": 0.17738628916812832, + "nauc_precision_at_1_diff1": 0.3101579718783373, + "nauc_precision_at_1_max": 0.232794151385547, + "nauc_precision_at_1_std": 0.06835199024544518, + "nauc_precision_at_20_diff1": 0.1769881933121576, + "nauc_precision_at_20_max": 0.3372188274771704, + "nauc_precision_at_20_std": 0.22139915186382111, + "nauc_precision_at_3_diff1": 0.2285147788071076, + "nauc_precision_at_3_max": 0.25658700565751275, + "nauc_precision_at_3_std": 0.11018748578571341, + "nauc_precision_at_5_diff1": 0.20566479129328738, + "nauc_precision_at_5_max": 0.2839102946088296, + "nauc_precision_at_5_std": 0.12812964861509582, + "nauc_recall_at_1000_diff1": 0.26283950052010496, + "nauc_recall_at_1000_max": 0.5980696173061514, + "nauc_recall_at_1000_std": 0.6186506542064352, + "nauc_recall_at_100_diff1": 0.2526269647152485, + "nauc_recall_at_100_max": 0.4567630585012785, + "nauc_recall_at_100_std": 0.3565671831986534, + "nauc_recall_at_10_diff1": 0.22822397160742405, + "nauc_recall_at_10_max": 0.32691113725390125, + "nauc_recall_at_10_std": 0.15467217375870954, + "nauc_recall_at_1_diff1": 0.31563754317382775, + "nauc_recall_at_1_max": 0.22668271138365942, + "nauc_recall_at_1_std": 0.055208961001451236, + "nauc_recall_at_20_diff1": 0.23344737100376814, + "nauc_recall_at_20_max": 0.365902020750824, + "nauc_recall_at_20_std": 0.20992710816215207, + "nauc_recall_at_3_diff1": 0.23742653189575516, + "nauc_recall_at_3_max": 0.24932858486907053, + "nauc_recall_at_3_std": 0.0953152035180649, + "nauc_recall_at_5_diff1": 0.2164483351896991, + "nauc_recall_at_5_max": 0.2754227548267417, + "nauc_recall_at_5_std": 0.11141679577931847, + "ndcg_at_1": 0.15093, + "ndcg_at_10": 0.285, + "ndcg_at_100": 0.35064, + "ndcg_at_1000": 0.37073, + "ndcg_at_20": 0.31292, + "ndcg_at_3": 0.21787, + "ndcg_at_5": 0.25096, + "precision_at_1": 0.15093, + "precision_at_10": 0.05211, + "precision_at_100": 0.00888, + "precision_at_1000": 0.00108, + "precision_at_20": 0.03253, + "precision_at_3": 0.10236, + "precision_at_5": 0.07978, + "recall_at_1": 0.13526, + "recall_at_10": 0.44619, + "recall_at_100": 0.74329, + "recall_at_1000": 0.89518, + "recall_at_20": 0.55065, + "recall_at_3": 0.26866, + "recall_at_5": 0.34618 + } + ] + }, + "task_name": "NQ" +} \ No newline at end of file diff --git a/results/bm25s/0_1_10/QuoraRetrieval.json b/results/bm25s/0_1_10/QuoraRetrieval.json new file mode 100644 index 0000000000..59d58f2017 --- /dev/null +++ b/results/bm25s/0_1_10/QuoraRetrieval.json @@ -0,0 +1,307 @@ +{ + "dataset_revision": "e4e08e0b7dbe3c8700f0daef558ff32256715259", + "evaluation_time": 131.51577734947205, + "kg_co2_emissions": null, + "mteb_version": "1.12.22", + "scores": { + "dev": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.80243, + "map_at_1": 0.62401, + "map_at_10": 0.75403, + "map_at_100": 0.76081, + "map_at_1000": 0.76112, + "map_at_20": 0.75839, + "map_at_3": 0.72273, + "map_at_5": 0.74182, + "mrr_at_1": 0.7172, + "mrr_at_10": 0.795938650793652, + "mrr_at_100": 0.7981634681320836, + "mrr_at_1000": 0.7982097042344083, + "mrr_at_20": 0.7974626707339797, + "mrr_at_3": 0.7804666666666689, + "mrr_at_5": 0.7907066666666679, + "nauc_map_at_1000_diff1": 0.6862606573373667, + "nauc_map_at_1000_max": 0.37957710557901697, + "nauc_map_at_1000_std": -0.08190887740041565, + "nauc_map_at_100_diff1": 0.6862727515901894, + "nauc_map_at_100_max": 0.37925144033419195, + "nauc_map_at_100_std": -0.0824051206314168, + "nauc_map_at_10_diff1": 0.6848406243930852, + "nauc_map_at_10_max": 0.37538375993662443, + "nauc_map_at_10_std": -0.09035989976028713, + "nauc_map_at_1_diff1": 0.702693751012232, + "nauc_map_at_1_max": 0.3063115880746668, + "nauc_map_at_1_std": -0.10028645236032527, + "nauc_map_at_20_diff1": 0.6858051564880256, + "nauc_map_at_20_max": 0.3778481671210552, + "nauc_map_at_20_std": -0.08562069538236837, + "nauc_map_at_3_diff1": 0.6810049716669562, + "nauc_map_at_3_max": 0.3578263514524082, + "nauc_map_at_3_std": -0.09987362210441189, + "nauc_map_at_5_diff1": 0.683109693875332, + "nauc_map_at_5_max": 0.3687718062700439, + "nauc_map_at_5_std": -0.097130174962681, + "nauc_mrr_at_1000_diff1": 0.7001676931993724, + "nauc_mrr_at_1000_max": 0.39512293634167023, + "nauc_mrr_at_1000_std": -0.060595527238305184, + "nauc_mrr_at_100_diff1": 0.7001973702897958, + "nauc_mrr_at_100_max": 0.39518782919559536, + "nauc_mrr_at_100_std": -0.06049796468503168, + "nauc_mrr_at_10_diff1": 0.6992005134316728, + "nauc_mrr_at_10_max": 0.3950170536921685, + "nauc_mrr_at_10_std": -0.06100649349631261, + "nauc_mrr_at_1_diff1": 0.7116699678859605, + "nauc_mrr_at_1_max": 0.3805369848830146, + "nauc_mrr_at_1_std": -0.061969575296938965, + "nauc_mrr_at_20_diff1": 0.7001058545009585, + "nauc_mrr_at_20_max": 0.3951763722401683, + "nauc_mrr_at_20_std": -0.060871451535432754, + "nauc_mrr_at_3_diff1": 0.6994248382342779, + "nauc_mrr_at_3_max": 0.3949500240011292, + "nauc_mrr_at_3_std": -0.06303289625738259, + "nauc_mrr_at_5_diff1": 0.6965816138063905, + "nauc_mrr_at_5_max": 0.3929371558596262, + "nauc_mrr_at_5_std": -0.06452877252736908, + "nauc_ndcg_at_1000_diff1": 0.6901559028612276, + "nauc_ndcg_at_1000_max": 0.3957478852775638, + "nauc_ndcg_at_1000_std": -0.061806307012296655, + "nauc_ndcg_at_100_diff1": 0.690358940492306, + "nauc_ndcg_at_100_max": 0.39583149215361035, + "nauc_ndcg_at_100_std": -0.06229978088613807, + "nauc_ndcg_at_10_diff1": 0.6848325423744238, + "nauc_ndcg_at_10_max": 0.3901881366861685, + "nauc_ndcg_at_10_std": -0.08167409061901061, + "nauc_ndcg_at_1_diff1": 0.7163646147983742, + "nauc_ndcg_at_1_max": 0.38166576601020197, + "nauc_ndcg_at_1_std": -0.05654218567152029, + "nauc_ndcg_at_20_diff1": 0.6879258868938293, + "nauc_ndcg_at_20_max": 0.3929190981813396, + "nauc_ndcg_at_20_std": -0.07403659758802644, + "nauc_ndcg_at_3_diff1": 0.6781874733166562, + "nauc_ndcg_at_3_max": 0.3795885253636755, + "nauc_ndcg_at_3_std": -0.08255270720195974, + "nauc_ndcg_at_5_diff1": 0.6809777027093813, + "nauc_ndcg_at_5_max": 0.38401090292812007, + "nauc_ndcg_at_5_std": -0.08859796125291687, + "nauc_precision_at_1000_diff1": -0.3039650539140641, + "nauc_precision_at_1000_max": -0.0023755241505917513, + "nauc_precision_at_1000_std": 0.21525129442945626, + "nauc_precision_at_100_diff1": -0.27345125770166717, + "nauc_precision_at_100_max": 0.011166930666594493, + "nauc_precision_at_100_std": 0.20236150997124924, + "nauc_precision_at_10_diff1": -0.15352908667758572, + "nauc_precision_at_10_max": 0.08295416426306115, + "nauc_precision_at_10_std": 0.12088425210213488, + "nauc_precision_at_1_diff1": 0.7163646147983742, + "nauc_precision_at_1_max": 0.38166576601020197, + "nauc_precision_at_1_std": -0.05654218567152029, + "nauc_precision_at_20_diff1": -0.21004701742045695, + "nauc_precision_at_20_max": 0.04844974686667347, + "nauc_precision_at_20_std": 0.15424215660794344, + "nauc_precision_at_3_diff1": 0.10654036800697735, + "nauc_precision_at_3_max": 0.20804005707694764, + "nauc_precision_at_3_std": 0.04560788755944524, + "nauc_precision_at_5_diff1": -0.034092761720762615, + "nauc_precision_at_5_max": 0.1455789354363942, + "nauc_precision_at_5_std": 0.0774480568776462, + "nauc_recall_at_1000_diff1": 0.6660273100257105, + "nauc_recall_at_1000_max": 0.7562707263246745, + "nauc_recall_at_1000_std": 0.604796550273321, + "nauc_recall_at_100_diff1": 0.6576517866878142, + "nauc_recall_at_100_max": 0.5029521187030306, + "nauc_recall_at_100_std": 0.13325208433936603, + "nauc_recall_at_10_diff1": 0.6126384134891587, + "nauc_recall_at_10_max": 0.38961392929479877, + "nauc_recall_at_10_std": -0.11577494123125191, + "nauc_recall_at_1_diff1": 0.702693751012232, + "nauc_recall_at_1_max": 0.3063115880746668, + "nauc_recall_at_1_std": -0.10028645236032527, + "nauc_recall_at_20_diff1": 0.6204199755722092, + "nauc_recall_at_20_max": 0.40751753853116623, + "nauc_recall_at_20_std": -0.08278130603048266, + "nauc_recall_at_3_diff1": 0.6315519546194731, + "nauc_recall_at_3_max": 0.3471732714846572, + "nauc_recall_at_3_std": -0.12469199834458174, + "nauc_recall_at_5_diff1": 0.6170611004939214, + "nauc_recall_at_5_max": 0.36462021586226373, + "nauc_recall_at_5_std": -0.13325117393579067, + "ndcg_at_1": 0.7148, + "ndcg_at_10": 0.80243, + "ndcg_at_100": 0.82186, + "ndcg_at_1000": 0.82559, + "ndcg_at_20": 0.81201, + "ndcg_at_3": 0.76329, + "ndcg_at_5": 0.78376, + "precision_at_1": 0.7148, + "precision_at_10": 0.12168, + "precision_at_100": 0.01418, + "precision_at_1000": 0.0015, + "precision_at_20": 0.0649, + "precision_at_3": 0.33107, + "precision_at_5": 0.21992, + "recall_at_1": 0.62401, + "recall_at_10": 0.90018, + "recall_at_100": 0.974, + "recall_at_1000": 0.99498, + "recall_at_20": 0.93225, + "recall_at_3": 0.79111, + "recall_at_5": 0.84667 + } + ], + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.80424, + "map_at_1": 0.6255, + "map_at_10": 0.75529, + "map_at_100": 0.7626, + "map_at_1000": 0.76292, + "map_at_20": 0.75987, + "map_at_3": 0.72481, + "map_at_5": 0.74318, + "mrr_at_1": 0.7191, + "mrr_at_10": 0.7990055952380904, + "mrr_at_100": 0.8013136691992537, + "mrr_at_1000": 0.801351063320376, + "mrr_at_20": 0.8005933852164291, + "mrr_at_3": 0.7836499999999957, + "mrr_at_5": 0.7937649999999934, + "nauc_map_at_1000_diff1": 0.6964268451101864, + "nauc_map_at_1000_max": 0.3875285786715886, + "nauc_map_at_1000_std": -0.09229889071886138, + "nauc_map_at_100_diff1": 0.6963460701660319, + "nauc_map_at_100_max": 0.3872882951146747, + "nauc_map_at_100_std": -0.0927004879351845, + "nauc_map_at_10_diff1": 0.6956904804789744, + "nauc_map_at_10_max": 0.38215276865692044, + "nauc_map_at_10_std": -0.10142554631183781, + "nauc_map_at_1_diff1": 0.7133554657311546, + "nauc_map_at_1_max": 0.32895547838136985, + "nauc_map_at_1_std": -0.09184989600149596, + "nauc_map_at_20_diff1": 0.6960604225569085, + "nauc_map_at_20_max": 0.3855968829659799, + "nauc_map_at_20_std": -0.09638744573204205, + "nauc_map_at_3_diff1": 0.6936725073120443, + "nauc_map_at_3_max": 0.3644515249363498, + "nauc_map_at_3_std": -0.11289942041539007, + "nauc_map_at_5_diff1": 0.6949923725099652, + "nauc_map_at_5_max": 0.37485987612943084, + "nauc_map_at_5_std": -0.10759796788262087, + "nauc_mrr_at_1000_diff1": 0.7090418423653326, + "nauc_mrr_at_1000_max": 0.406977478346614, + "nauc_mrr_at_1000_std": -0.06891365994780754, + "nauc_mrr_at_100_diff1": 0.7090411812375309, + "nauc_mrr_at_100_max": 0.4070125732052164, + "nauc_mrr_at_100_std": -0.06888119521147974, + "nauc_mrr_at_10_diff1": 0.7085680439782271, + "nauc_mrr_at_10_max": 0.40726741571472225, + "nauc_mrr_at_10_std": -0.07001862292047163, + "nauc_mrr_at_1_diff1": 0.7226694660074746, + "nauc_mrr_at_1_max": 0.39538405597311715, + "nauc_mrr_at_1_std": -0.059874535797864434, + "nauc_mrr_at_20_diff1": 0.7089189883912925, + "nauc_mrr_at_20_max": 0.40746459037284105, + "nauc_mrr_at_20_std": -0.0691004585022329, + "nauc_mrr_at_3_diff1": 0.7063149709458826, + "nauc_mrr_at_3_max": 0.40414859740947556, + "nauc_mrr_at_3_std": -0.07451763060819545, + "nauc_mrr_at_5_diff1": 0.7069961917001073, + "nauc_mrr_at_5_max": 0.4061218716125118, + "nauc_mrr_at_5_std": -0.0729108882958794, + "nauc_ndcg_at_1000_diff1": 0.6985663552123943, + "nauc_ndcg_at_1000_max": 0.40142222968026975, + "nauc_ndcg_at_1000_std": -0.07732477286636358, + "nauc_ndcg_at_100_diff1": 0.6976597963860574, + "nauc_ndcg_at_100_max": 0.401459894855972, + "nauc_ndcg_at_100_std": -0.07813995040302399, + "nauc_ndcg_at_10_diff1": 0.6931202728344945, + "nauc_ndcg_at_10_max": 0.3984374169533205, + "nauc_ndcg_at_10_std": -0.09484205695408379, + "nauc_ndcg_at_1_diff1": 0.7280806229440068, + "nauc_ndcg_at_1_max": 0.39465569721971183, + "nauc_ndcg_at_1_std": -0.05883439749726324, + "nauc_ndcg_at_20_diff1": 0.6954388406230854, + "nauc_ndcg_at_20_max": 0.4018054891600623, + "nauc_ndcg_at_20_std": -0.08797805825743059, + "nauc_ndcg_at_3_diff1": 0.6873196345755627, + "nauc_ndcg_at_3_max": 0.3843942374676488, + "nauc_ndcg_at_3_std": -0.10234276640859721, + "nauc_ndcg_at_5_diff1": 0.6889675467311537, + "nauc_ndcg_at_5_max": 0.39104807332551383, + "nauc_ndcg_at_5_std": -0.09996550644215504, + "nauc_precision_at_1000_diff1": -0.3225387188189387, + "nauc_precision_at_1000_max": -0.024434461716359542, + "nauc_precision_at_1000_std": 0.20415554611153522, + "nauc_precision_at_100_diff1": -0.3014167618789836, + "nauc_precision_at_100_max": -0.009586322883204913, + "nauc_precision_at_100_std": 0.19502305291839334, + "nauc_precision_at_10_diff1": -0.1711978299853412, + "nauc_precision_at_10_max": 0.06843380802724298, + "nauc_precision_at_10_std": 0.10818261945042076, + "nauc_precision_at_1_diff1": 0.7280806229440068, + "nauc_precision_at_1_max": 0.39465569721971183, + "nauc_precision_at_1_std": -0.05883439749726324, + "nauc_precision_at_20_diff1": -0.23576393663492554, + "nauc_precision_at_20_max": 0.03803719274372207, + "nauc_precision_at_20_std": 0.15097703014824404, + "nauc_precision_at_3_diff1": 0.11013188132106194, + "nauc_precision_at_3_max": 0.17939807606438765, + "nauc_precision_at_3_std": -0.00010661554750169979, + "nauc_precision_at_5_diff1": -0.04082215254047251, + "nauc_precision_at_5_max": 0.12312274622727233, + "nauc_precision_at_5_std": 0.05232111594909391, + "nauc_recall_at_1000_diff1": 0.6102028652533463, + "nauc_recall_at_1000_max": 0.5745623875910069, + "nauc_recall_at_1000_std": 0.3728601194709307, + "nauc_recall_at_100_diff1": 0.6025289522659478, + "nauc_recall_at_100_max": 0.4652653504060352, + "nauc_recall_at_100_std": 0.06570517160755485, + "nauc_recall_at_10_diff1": 0.6153760135496282, + "nauc_recall_at_10_max": 0.3905165650179025, + "nauc_recall_at_10_std": -0.14371417612918225, + "nauc_recall_at_1_diff1": 0.7133554657311546, + "nauc_recall_at_1_max": 0.32895547838136985, + "nauc_recall_at_1_std": -0.09184989600149596, + "nauc_recall_at_20_diff1": 0.6065993236691276, + "nauc_recall_at_20_max": 0.4259250536711636, + "nauc_recall_at_20_std": -0.10233132777344608, + "nauc_recall_at_3_diff1": 0.6373593133043983, + "nauc_recall_at_3_max": 0.34596324337780016, + "nauc_recall_at_3_std": -0.15299533049116804, + "nauc_recall_at_5_diff1": 0.6222877404220493, + "nauc_recall_at_5_max": 0.3596029280436281, + "nauc_recall_at_5_std": -0.15483100636291305, + "ndcg_at_1": 0.7163, + "ndcg_at_10": 0.80424, + "ndcg_at_100": 0.82455, + "ndcg_at_1000": 0.82799, + "ndcg_at_20": 0.81402, + "ndcg_at_3": 0.76573, + "ndcg_at_5": 0.78597, + "precision_at_1": 0.7163, + "precision_at_10": 0.12165, + "precision_at_100": 0.01446, + "precision_at_1000": 0.00154, + "precision_at_20": 0.06526, + "precision_at_3": 0.33037, + "precision_at_5": 0.21948, + "recall_at_1": 0.6255, + "recall_at_10": 0.90101, + "recall_at_100": 0.97713, + "recall_at_1000": 0.99542, + "recall_at_20": 0.93404, + "recall_at_3": 0.79439, + "recall_at_5": 0.84832 + } + ] + }, + "task_name": "QuoraRetrieval" +} \ No newline at end of file diff --git a/results/bm25s/0_1_10/SCIDOCS.json b/results/bm25s/0_1_10/SCIDOCS.json new file mode 100644 index 0000000000..6a71d9915a --- /dev/null +++ b/results/bm25s/0_1_10/SCIDOCS.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88", + "evaluation_time": 9.77201247215271, + "kg_co2_emissions": null, + "mteb_version": "1.12.22", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.15783, + "map_at_1": 0.03723, + "map_at_10": 0.09233, + "map_at_100": 0.10779, + "map_at_1000": 0.11005, + "map_at_20": 0.09979, + "map_at_3": 0.06752, + "map_at_5": 0.07959, + "mrr_at_1": 0.183, + "mrr_at_10": 0.2746146825396823, + "mrr_at_100": 0.2863799795494601, + "mrr_at_1000": 0.28704566726442926, + "mrr_at_20": 0.2820460117410967, + "mrr_at_3": 0.24733333333333365, + "mrr_at_5": 0.26213333333333333, + "nauc_map_at_1000_diff1": 0.24624364170339427, + "nauc_map_at_1000_max": 0.42797057412536393, + "nauc_map_at_1000_std": 0.2673925482237281, + "nauc_map_at_100_diff1": 0.2455664222816774, + "nauc_map_at_100_max": 0.42545060399547063, + "nauc_map_at_100_std": 0.2642770340072519, + "nauc_map_at_10_diff1": 0.24359791081397256, + "nauc_map_at_10_max": 0.40524370951637034, + "nauc_map_at_10_std": 0.23480591990987407, + "nauc_map_at_1_diff1": 0.28616868693925385, + "nauc_map_at_1_max": 0.30351747196145273, + "nauc_map_at_1_std": 0.15823815561750998, + "nauc_map_at_20_diff1": 0.2447637684512533, + "nauc_map_at_20_max": 0.41365613602552354, + "nauc_map_at_20_std": 0.2441455852487958, + "nauc_map_at_3_diff1": 0.26751741291543746, + "nauc_map_at_3_max": 0.3431994827300431, + "nauc_map_at_3_std": 0.1679050167038632, + "nauc_map_at_5_diff1": 0.25767914787922397, + "nauc_map_at_5_max": 0.3854015244905599, + "nauc_map_at_5_std": 0.20480508862691169, + "nauc_mrr_at_1000_diff1": 0.24555452170951902, + "nauc_mrr_at_1000_max": 0.3330324164024491, + "nauc_mrr_at_1000_std": 0.19277002630591752, + "nauc_mrr_at_100_diff1": 0.2451702857916294, + "nauc_mrr_at_100_max": 0.3330325574342476, + "nauc_mrr_at_100_std": 0.19305129261591505, + "nauc_mrr_at_10_diff1": 0.2452837999379923, + "nauc_mrr_at_10_max": 0.3319088265493578, + "nauc_mrr_at_10_std": 0.18904749551079775, + "nauc_mrr_at_1_diff1": 0.2836711050616406, + "nauc_mrr_at_1_max": 0.3074839686143307, + "nauc_mrr_at_1_std": 0.16418251622187474, + "nauc_mrr_at_20_diff1": 0.24449910518679227, + "nauc_mrr_at_20_max": 0.3315575609996607, + "nauc_mrr_at_20_std": 0.1904479683369985, + "nauc_mrr_at_3_diff1": 0.251089395466814, + "nauc_mrr_at_3_max": 0.322469442050665, + "nauc_mrr_at_3_std": 0.1761078399889434, + "nauc_mrr_at_5_diff1": 0.24680621909244285, + "nauc_mrr_at_5_max": 0.3306549639996516, + "nauc_mrr_at_5_std": 0.18621470530359982, + "nauc_ndcg_at_1000_diff1": 0.2323652082388895, + "nauc_ndcg_at_1000_max": 0.45885295272624044, + "nauc_ndcg_at_1000_std": 0.3369902894103974, + "nauc_ndcg_at_100_diff1": 0.22430590326895083, + "nauc_ndcg_at_100_max": 0.436155032045136, + "nauc_ndcg_at_100_std": 0.310310586035536, + "nauc_ndcg_at_10_diff1": 0.22270080734164194, + "nauc_ndcg_at_10_max": 0.3961118185561924, + "nauc_ndcg_at_10_std": 0.24120712736865849, + "nauc_ndcg_at_1_diff1": 0.2836711050616406, + "nauc_ndcg_at_1_max": 0.3074839686143307, + "nauc_ndcg_at_1_std": 0.16418251622187474, + "nauc_ndcg_at_20_diff1": 0.22155569639680509, + "nauc_ndcg_at_20_max": 0.4056316362319183, + "nauc_ndcg_at_20_std": 0.2552466270895776, + "nauc_ndcg_at_3_diff1": 0.25413618630330825, + "nauc_ndcg_at_3_max": 0.340485479474139, + "nauc_ndcg_at_3_std": 0.17667705685040036, + "nauc_ndcg_at_5_diff1": 0.24102642582318287, + "nauc_ndcg_at_5_max": 0.37690363753899186, + "nauc_ndcg_at_5_std": 0.2105640608742882, + "nauc_precision_at_1000_diff1": 0.15938728848601758, + "nauc_precision_at_1000_max": 0.4703458816089248, + "nauc_precision_at_1000_std": 0.4534190875406407, + "nauc_precision_at_100_diff1": 0.16008567514138433, + "nauc_precision_at_100_max": 0.4439248536109304, + "nauc_precision_at_100_std": 0.3945009477753865, + "nauc_precision_at_10_diff1": 0.17052415277367627, + "nauc_precision_at_10_max": 0.4122062560963605, + "nauc_precision_at_10_std": 0.2786870117318588, + "nauc_precision_at_1_diff1": 0.2836711050616406, + "nauc_precision_at_1_max": 0.3074839686143307, + "nauc_precision_at_1_std": 0.16418251622187474, + "nauc_precision_at_20_diff1": 0.16519254599017574, + "nauc_precision_at_20_max": 0.4132803017261771, + "nauc_precision_at_20_std": 0.29352272821480146, + "nauc_precision_at_3_diff1": 0.23966879648377623, + "nauc_precision_at_3_max": 0.34836869573985557, + "nauc_precision_at_3_std": 0.18005792550351685, + "nauc_precision_at_5_diff1": 0.2118295864868873, + "nauc_precision_at_5_max": 0.4026776963694378, + "nauc_precision_at_5_std": 0.23892964103095518, + "nauc_recall_at_1000_diff1": 0.15663332685354514, + "nauc_recall_at_1000_max": 0.47502874277468526, + "nauc_recall_at_1000_std": 0.45648982863869675, + "nauc_recall_at_100_diff1": 0.1571873943321865, + "nauc_recall_at_100_max": 0.44347913505704945, + "nauc_recall_at_100_std": 0.39304918747648054, + "nauc_recall_at_10_diff1": 0.17065392345949035, + "nauc_recall_at_10_max": 0.4094577459625101, + "nauc_recall_at_10_std": 0.27415307284664653, + "nauc_recall_at_1_diff1": 0.28616868693925385, + "nauc_recall_at_1_max": 0.30351747196145273, + "nauc_recall_at_1_std": 0.15823815561750998, + "nauc_recall_at_20_diff1": 0.16425797865795502, + "nauc_recall_at_20_max": 0.4107255065595408, + "nauc_recall_at_20_std": 0.28949497911333905, + "nauc_recall_at_3_diff1": 0.23897041378939468, + "nauc_recall_at_3_max": 0.34438021339819225, + "nauc_recall_at_3_std": 0.17396798357680937, + "nauc_recall_at_5_diff1": 0.21280603158192724, + "nauc_recall_at_5_max": 0.3996507674176187, + "nauc_recall_at_5_std": 0.23321453162574626, + "ndcg_at_1": 0.183, + "ndcg_at_10": 0.15783, + "ndcg_at_100": 0.22519, + "ndcg_at_1000": 0.26848, + "ndcg_at_20": 0.18108, + "ndcg_at_3": 0.15236, + "ndcg_at_5": 0.13137, + "precision_at_1": 0.183, + "precision_at_10": 0.0816, + "precision_at_100": 0.01796, + "precision_at_1000": 0.00283, + "precision_at_20": 0.05425, + "precision_at_3": 0.14367, + "precision_at_5": 0.1152, + "recall_at_1": 0.03723, + "recall_at_10": 0.16528, + "recall_at_100": 0.36478, + "recall_at_1000": 0.5752, + "recall_at_20": 0.21977, + "recall_at_3": 0.08738, + "recall_at_5": 0.11678 + } + ] + }, + "task_name": "SCIDOCS" +} \ No newline at end of file diff --git a/results/bm25s/0_1_10/SciFact.json b/results/bm25s/0_1_10/SciFact.json new file mode 100644 index 0000000000..9d39c4c764 --- /dev/null +++ b/results/bm25s/0_1_10/SciFact.json @@ -0,0 +1,307 @@ +{ + "dataset_revision": "0228b52cf27578f30900b9e5271d331663a030d7", + "evaluation_time": 5.407999515533447, + "kg_co2_emissions": null, + "mteb_version": "1.12.22", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.68703, + "map_at_1": 0.5375, + "map_at_10": 0.64011, + "map_at_100": 0.64542, + "map_at_1000": 0.64569, + "map_at_20": 0.64349, + "map_at_3": 0.61977, + "map_at_5": 0.62702, + "mrr_at_1": 0.5533333333333333, + "mrr_at_10": 0.6503399470899471, + "mrr_at_100": 0.6539190102920337, + "mrr_at_1000": 0.6541495415725362, + "mrr_at_20": 0.6527961835922362, + "mrr_at_3": 0.6327777777777779, + "mrr_at_5": 0.6396111111111111, + "nauc_map_at_1000_diff1": 0.6515802163337553, + "nauc_map_at_1000_max": 0.44869406537117645, + "nauc_map_at_1000_std": 0.13440242926811222, + "nauc_map_at_100_diff1": 0.6516203583186828, + "nauc_map_at_100_max": 0.4488480388820315, + "nauc_map_at_100_std": 0.13442409481123158, + "nauc_map_at_10_diff1": 0.6513749958292991, + "nauc_map_at_10_max": 0.4532468698751438, + "nauc_map_at_10_std": 0.13734686225094148, + "nauc_map_at_1_diff1": 0.6679690792474182, + "nauc_map_at_1_max": 0.39641028305535836, + "nauc_map_at_1_std": 0.07946217749989873, + "nauc_map_at_20_diff1": 0.6519782394855551, + "nauc_map_at_20_max": 0.4493581021265059, + "nauc_map_at_20_std": 0.1343347487606731, + "nauc_map_at_3_diff1": 0.6560016534579957, + "nauc_map_at_3_max": 0.4298358265914772, + "nauc_map_at_3_std": 0.10348594090142527, + "nauc_map_at_5_diff1": 0.6479934385384846, + "nauc_map_at_5_max": 0.43417707010895945, + "nauc_map_at_5_std": 0.11511836002527605, + "nauc_mrr_at_1000_diff1": 0.6567344725515428, + "nauc_mrr_at_1000_max": 0.44959154831734127, + "nauc_mrr_at_1000_std": 0.1395470543544745, + "nauc_mrr_at_100_diff1": 0.6567281068188324, + "nauc_mrr_at_100_max": 0.4496629315373417, + "nauc_mrr_at_100_std": 0.1394478390522366, + "nauc_mrr_at_10_diff1": 0.6558204049912286, + "nauc_mrr_at_10_max": 0.4537917968815809, + "nauc_mrr_at_10_std": 0.14500662706345036, + "nauc_mrr_at_1_diff1": 0.6711120923721567, + "nauc_mrr_at_1_max": 0.39838780897979115, + "nauc_mrr_at_1_std": 0.08391660951343036, + "nauc_mrr_at_20_diff1": 0.6569207385241228, + "nauc_mrr_at_20_max": 0.4503977789550144, + "nauc_mrr_at_20_std": 0.14106517542445376, + "nauc_mrr_at_3_diff1": 0.6616896267398721, + "nauc_mrr_at_3_max": 0.4461022666465369, + "nauc_mrr_at_3_std": 0.12807852729168248, + "nauc_mrr_at_5_diff1": 0.6523783762120482, + "nauc_mrr_at_5_max": 0.4448604021760145, + "nauc_mrr_at_5_std": 0.13494088263597187, + "nauc_ndcg_at_1000_diff1": 0.649891012116661, + "nauc_ndcg_at_1000_max": 0.46884645959013155, + "nauc_ndcg_at_1000_std": 0.16452684883824967, + "nauc_ndcg_at_100_diff1": 0.6510361257449979, + "nauc_ndcg_at_100_max": 0.47217754440929155, + "nauc_ndcg_at_100_std": 0.16072946025712687, + "nauc_ndcg_at_10_diff1": 0.6478157913560237, + "nauc_ndcg_at_10_max": 0.4878962642548131, + "nauc_ndcg_at_10_std": 0.17763229348219722, + "nauc_ndcg_at_1_diff1": 0.6711120923721567, + "nauc_ndcg_at_1_max": 0.39838780897979115, + "nauc_ndcg_at_1_std": 0.08391660951343036, + "nauc_ndcg_at_20_diff1": 0.6519536856239165, + "nauc_ndcg_at_20_max": 0.4748422977894064, + "nauc_ndcg_at_20_std": 0.16683263779056515, + "nauc_ndcg_at_3_diff1": 0.6539730052084252, + "nauc_ndcg_at_3_max": 0.44958372840357136, + "nauc_ndcg_at_3_std": 0.12290050459229948, + "nauc_ndcg_at_5_diff1": 0.6400396731997274, + "nauc_ndcg_at_5_max": 0.45152743291495173, + "nauc_ndcg_at_5_std": 0.13669242274402937, + "nauc_precision_at_1000_diff1": -0.1741166608961855, + "nauc_precision_at_1000_max": 0.09760008746833829, + "nauc_precision_at_1000_std": 0.3349247711548868, + "nauc_precision_at_100_diff1": -0.0012518511415821184, + "nauc_precision_at_100_max": 0.21627193403010742, + "nauc_precision_at_100_std": 0.31556770117514144, + "nauc_precision_at_10_diff1": 0.2660783795886963, + "nauc_precision_at_10_max": 0.4929038110438742, + "nauc_precision_at_10_std": 0.42594357496354296, + "nauc_precision_at_1_diff1": 0.6711120923721567, + "nauc_precision_at_1_max": 0.39838780897979115, + "nauc_precision_at_1_std": 0.08391660951343036, + "nauc_precision_at_20_diff1": 0.16790692446043115, + "nauc_precision_at_20_max": 0.35787994604316503, + "nauc_precision_at_20_std": 0.3756575989208631, + "nauc_precision_at_3_diff1": 0.5238136652715113, + "nauc_precision_at_3_max": 0.47676795129640714, + "nauc_precision_at_3_std": 0.21367614284937766, + "nauc_precision_at_5_diff1": 0.41846182709073704, + "nauc_precision_at_5_max": 0.47076610678261627, + "nauc_precision_at_5_std": 0.2941708786721696, + "nauc_recall_at_1000_diff1": 0.6382404813777393, + "nauc_recall_at_1000_max": 0.767143894594878, + "nauc_recall_at_1000_std": 0.7393920531175461, + "nauc_recall_at_100_diff1": 0.6664225486632297, + "nauc_recall_at_100_max": 0.6344288351472888, + "nauc_recall_at_100_std": 0.2668530516532301, + "nauc_recall_at_10_diff1": 0.6221035349956205, + "nauc_recall_at_10_max": 0.6356568652035337, + "nauc_recall_at_10_std": 0.3346107579870839, + "nauc_recall_at_1_diff1": 0.6679690792474182, + "nauc_recall_at_1_max": 0.39641028305535836, + "nauc_recall_at_1_std": 0.07946217749989873, + "nauc_recall_at_20_diff1": 0.6579383427854262, + "nauc_recall_at_20_max": 0.5917346871372899, + "nauc_recall_at_20_std": 0.2978794598624182, + "nauc_recall_at_3_diff1": 0.6409803692853794, + "nauc_recall_at_3_max": 0.4773008139940585, + "nauc_recall_at_3_std": 0.13382625509806087, + "nauc_recall_at_5_diff1": 0.60153758879396, + "nauc_recall_at_5_max": 0.48376717492159743, + "nauc_recall_at_5_std": 0.17232963559800124, + "ndcg_at_1": 0.55333, + "ndcg_at_10": 0.68703, + "ndcg_at_100": 0.70892, + "ndcg_at_1000": 0.71624, + "ndcg_at_20": 0.69777, + "ndcg_at_3": 0.64726, + "ndcg_at_5": 0.65872, + "precision_at_1": 0.55333, + "precision_at_10": 0.09033, + "precision_at_100": 0.01037, + "precision_at_1000": 0.0011, + "precision_at_20": 0.04783, + "precision_at_3": 0.25111, + "precision_at_5": 0.15867, + "recall_at_1": 0.5375, + "recall_at_10": 0.818, + "recall_at_100": 0.91267, + "recall_at_1000": 0.97, + "recall_at_20": 0.85694, + "recall_at_3": 0.70861, + "recall_at_5": 0.73622 + } + ], + "train": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.696, + "map_at_1": 0.56875, + "map_at_10": 0.65333, + "map_at_100": 0.65939, + "map_at_1000": 0.65962, + "map_at_20": 0.65725, + "map_at_3": 0.62672, + "map_at_5": 0.64286, + "mrr_at_1": 0.5920889987639061, + "mrr_at_10": 0.6656463986501069, + "mrr_at_100": 0.670029120961771, + "mrr_at_1000": 0.6702201896259565, + "mrr_at_20": 0.6685074879992159, + "mrr_at_3": 0.6442109600329622, + "mrr_at_5": 0.65613926658426, + "nauc_map_at_1000_diff1": 0.7239712678815537, + "nauc_map_at_1000_max": 0.5208963663782994, + "nauc_map_at_1000_std": 0.16964367110250128, + "nauc_map_at_100_diff1": 0.7238431951036898, + "nauc_map_at_100_max": 0.5209897773881522, + "nauc_map_at_100_std": 0.16971443003477418, + "nauc_map_at_10_diff1": 0.7231623562379834, + "nauc_map_at_10_max": 0.520783949212097, + "nauc_map_at_10_std": 0.16891414035728605, + "nauc_map_at_1_diff1": 0.7428811189742983, + "nauc_map_at_1_max": 0.46204719633339164, + "nauc_map_at_1_std": 0.1044692010324822, + "nauc_map_at_20_diff1": 0.7230958122051122, + "nauc_map_at_20_max": 0.5208006301263902, + "nauc_map_at_20_std": 0.16967545162084138, + "nauc_map_at_3_diff1": 0.72510134429939, + "nauc_map_at_3_max": 0.499616775389838, + "nauc_map_at_3_std": 0.13958054450883162, + "nauc_map_at_5_diff1": 0.7229198857950043, + "nauc_map_at_5_max": 0.506717761464418, + "nauc_map_at_5_std": 0.15225614235532645, + "nauc_mrr_at_1000_diff1": 0.7176419736643064, + "nauc_mrr_at_1000_max": 0.5188116751448882, + "nauc_mrr_at_1000_std": 0.17158951699372815, + "nauc_mrr_at_100_diff1": 0.7176307688481897, + "nauc_mrr_at_100_max": 0.5189260110249083, + "nauc_mrr_at_100_std": 0.17159566641675084, + "nauc_mrr_at_10_diff1": 0.7172811858269991, + "nauc_mrr_at_10_max": 0.5208846572368236, + "nauc_mrr_at_10_std": 0.17385161688453307, + "nauc_mrr_at_1_diff1": 0.732616156532563, + "nauc_mrr_at_1_max": 0.4830261104031932, + "nauc_mrr_at_1_std": 0.12684210547198305, + "nauc_mrr_at_20_diff1": 0.7173109277363299, + "nauc_mrr_at_20_max": 0.519446278449187, + "nauc_mrr_at_20_std": 0.17222508502085138, + "nauc_mrr_at_3_diff1": 0.721311900627719, + "nauc_mrr_at_3_max": 0.5177071077878291, + "nauc_mrr_at_3_std": 0.16380655475400407, + "nauc_mrr_at_5_diff1": 0.7170467952485343, + "nauc_mrr_at_5_max": 0.5118811288445246, + "nauc_mrr_at_5_std": 0.16395544170072995, + "nauc_ndcg_at_1000_diff1": 0.7196808496717253, + "nauc_ndcg_at_1000_max": 0.5380516342272229, + "nauc_ndcg_at_1000_std": 0.1952966175667907, + "nauc_ndcg_at_100_diff1": 0.717752870147442, + "nauc_ndcg_at_100_max": 0.5413055394605453, + "nauc_ndcg_at_100_std": 0.1980366513983387, + "nauc_ndcg_at_10_diff1": 0.7128875946183356, + "nauc_ndcg_at_10_max": 0.5432271154673403, + "nauc_ndcg_at_10_std": 0.1993930890336125, + "nauc_ndcg_at_1_diff1": 0.732616156532563, + "nauc_ndcg_at_1_max": 0.4830261104031932, + "nauc_ndcg_at_1_std": 0.12684210547198305, + "nauc_ndcg_at_20_diff1": 0.7133145230801325, + "nauc_ndcg_at_20_max": 0.5420460077204796, + "nauc_ndcg_at_20_std": 0.20009399642563544, + "nauc_ndcg_at_3_diff1": 0.718764055411564, + "nauc_ndcg_at_3_max": 0.5169657769743838, + "nauc_ndcg_at_3_std": 0.15772574785820184, + "nauc_ndcg_at_5_diff1": 0.7133180603693328, + "nauc_ndcg_at_5_max": 0.5151739926555841, + "nauc_ndcg_at_5_std": 0.16664716518445416, + "nauc_precision_at_1000_diff1": -0.09810177200813268, + "nauc_precision_at_1000_max": 0.17967554698525431, + "nauc_precision_at_1000_std": 0.2855149903610938, + "nauc_precision_at_100_diff1": 0.023439680712808372, + "nauc_precision_at_100_max": 0.2802451652973477, + "nauc_precision_at_100_std": 0.31964673321468146, + "nauc_precision_at_10_diff1": 0.29365835414981983, + "nauc_precision_at_10_max": 0.4977392162985132, + "nauc_precision_at_10_std": 0.3895559626079764, + "nauc_precision_at_1_diff1": 0.732616156532563, + "nauc_precision_at_1_max": 0.4830261104031932, + "nauc_precision_at_1_std": 0.12684210547198305, + "nauc_precision_at_20_diff1": 0.1968363856950331, + "nauc_precision_at_20_max": 0.4279903382804591, + "nauc_precision_at_20_std": 0.3819420473878428, + "nauc_precision_at_3_diff1": 0.5363779628662623, + "nauc_precision_at_3_max": 0.5118855197946177, + "nauc_precision_at_3_std": 0.23758038955435895, + "nauc_precision_at_5_diff1": 0.3991705721717818, + "nauc_precision_at_5_max": 0.45426954590664825, + "nauc_precision_at_5_std": 0.2711575247396718, + "nauc_recall_at_1000_diff1": 0.6773678546288053, + "nauc_recall_at_1000_max": 0.8123200361979269, + "nauc_recall_at_1000_std": 0.7061797873572301, + "nauc_recall_at_100_diff1": 0.6660477634563566, + "nauc_recall_at_100_max": 0.7125845683213293, + "nauc_recall_at_100_std": 0.4406169874180956, + "nauc_recall_at_10_diff1": 0.6510157138791036, + "nauc_recall_at_10_max": 0.6215012436895988, + "nauc_recall_at_10_std": 0.31364983766952137, + "nauc_recall_at_1_diff1": 0.7428811189742983, + "nauc_recall_at_1_max": 0.46204719633339164, + "nauc_recall_at_1_std": 0.1044692010324822, + "nauc_recall_at_20_diff1": 0.6441487798664844, + "nauc_recall_at_20_max": 0.6380917953927944, + "nauc_recall_at_20_std": 0.34872894240107977, + "nauc_recall_at_3_diff1": 0.6959044947397773, + "nauc_recall_at_3_max": 0.5288902462658955, + "nauc_recall_at_3_std": 0.16871639039506725, + "nauc_recall_at_5_diff1": 0.6747652469295257, + "nauc_recall_at_5_max": 0.5228287360808779, + "nauc_recall_at_5_std": 0.19085474745066683, + "ndcg_at_1": 0.59209, + "ndcg_at_10": 0.696, + "ndcg_at_100": 0.72189, + "ndcg_at_1000": 0.72801, + "ndcg_at_20": 0.7082, + "ndcg_at_3": 0.64855, + "ndcg_at_5": 0.6731, + "precision_at_1": 0.59209, + "precision_at_10": 0.09073, + "precision_at_100": 0.01056, + "precision_at_1000": 0.00111, + "precision_at_20": 0.04815, + "precision_at_3": 0.24887, + "precision_at_5": 0.16465, + "recall_at_1": 0.56875, + "recall_at_10": 0.81374, + "recall_at_100": 0.92827, + "recall_at_1000": 0.9761, + "recall_at_20": 0.8588, + "recall_at_3": 0.69112, + "recall_at_5": 0.74843 + } + ] + }, + "task_name": "SciFact" +} \ No newline at end of file diff --git a/results/bm25s/0_1_10/TRECCOVID.json b/results/bm25s/0_1_10/TRECCOVID.json new file mode 100644 index 0000000000..6501b4509e --- /dev/null +++ b/results/bm25s/0_1_10/TRECCOVID.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "bb9466bac8153a0349341eb1b22e06409e78ef4e", + "evaluation_time": 32.00744581222534, + "kg_co2_emissions": null, + "mteb_version": "1.12.22", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.62314, + "map_at_1": 0.00199, + "map_at_10": 0.01439, + "map_at_100": 0.07581, + "map_at_1000": 0.19485, + "map_at_20": 0.02414, + "map_at_3": 0.00546, + "map_at_5": 0.00832, + "mrr_at_1": 0.8, + "mrr_at_10": 0.8755555555555555, + "mrr_at_100": 0.8755555555555555, + "mrr_at_1000": 0.8755555555555555, + "mrr_at_20": 0.8755555555555555, + "mrr_at_3": 0.8733333333333334, + "mrr_at_5": 0.8733333333333334, + "nauc_map_at_1000_diff1": -0.17286809142447151, + "nauc_map_at_1000_max": 0.44412872746137255, + "nauc_map_at_1000_std": 0.6642438672266683, + "nauc_map_at_100_diff1": 0.02970616566865273, + "nauc_map_at_100_max": 0.43569861924655684, + "nauc_map_at_100_std": 0.6598150995516672, + "nauc_map_at_10_diff1": 0.15319545613672464, + "nauc_map_at_10_max": 0.3227101357717531, + "nauc_map_at_10_std": 0.4243223967736128, + "nauc_map_at_1_diff1": 0.25234611831096293, + "nauc_map_at_1_max": 0.19866732441765514, + "nauc_map_at_1_std": 0.21652983886626678, + "nauc_map_at_20_diff1": 0.15535979033692132, + "nauc_map_at_20_max": 0.37520494893642, + "nauc_map_at_20_std": 0.49323683759002745, + "nauc_map_at_3_diff1": 0.2040673874616903, + "nauc_map_at_3_max": 0.2729275441611162, + "nauc_map_at_3_std": 0.36531315504607625, + "nauc_map_at_5_diff1": 0.21781763451280806, + "nauc_map_at_5_max": 0.3291815240217156, + "nauc_map_at_5_std": 0.4168368798742171, + "nauc_mrr_at_1000_diff1": 0.16228627949696364, + "nauc_mrr_at_1000_max": 0.353627949696198, + "nauc_mrr_at_1000_std": 0.3539988695775061, + "nauc_mrr_at_100_diff1": 0.16228627949696364, + "nauc_mrr_at_100_max": 0.353627949696198, + "nauc_mrr_at_100_std": 0.3539988695775061, + "nauc_mrr_at_10_diff1": 0.16228627949696364, + "nauc_mrr_at_10_max": 0.353627949696198, + "nauc_mrr_at_10_std": 0.3539988695775061, + "nauc_mrr_at_1_diff1": 0.14735264735264755, + "nauc_mrr_at_1_max": 0.31168831168831174, + "nauc_mrr_at_1_std": 0.28791208791208733, + "nauc_mrr_at_20_diff1": 0.16228627949696364, + "nauc_mrr_at_20_max": 0.353627949696198, + "nauc_mrr_at_20_std": 0.3539988695775061, + "nauc_mrr_at_3_diff1": 0.1560923604971642, + "nauc_mrr_at_3_max": 0.34934213947683096, + "nauc_mrr_at_3_std": 0.3540745748608867, + "nauc_mrr_at_5_diff1": 0.1560923604971642, + "nauc_mrr_at_5_max": 0.34934213947683096, + "nauc_mrr_at_5_std": 0.3540745748608867, + "nauc_ndcg_at_1000_diff1": -0.10280049788921376, + "nauc_ndcg_at_1000_max": 0.47663111422435256, + "nauc_ndcg_at_1000_std": 0.6507783604002365, + "nauc_ndcg_at_100_diff1": -0.20359384304257597, + "nauc_ndcg_at_100_max": 0.3998182375977918, + "nauc_ndcg_at_100_std": 0.5979098896233466, + "nauc_ndcg_at_10_diff1": -0.14426908238631933, + "nauc_ndcg_at_10_max": 0.38875687922149327, + "nauc_ndcg_at_10_std": 0.5121514592168998, + "nauc_ndcg_at_1_diff1": 0.009950347963069725, + "nauc_ndcg_at_1_max": 0.32726475104189523, + "nauc_ndcg_at_1_std": 0.297613112923488, + "nauc_ndcg_at_20_diff1": -0.10374241556569612, + "nauc_ndcg_at_20_max": 0.4271593506653407, + "nauc_ndcg_at_20_std": 0.5599737071878734, + "nauc_ndcg_at_3_diff1": -0.1845402407891268, + "nauc_ndcg_at_3_max": 0.3108409069021974, + "nauc_ndcg_at_3_std": 0.4247085547493497, + "nauc_ndcg_at_5_diff1": -0.08623473850066488, + "nauc_ndcg_at_5_max": 0.40136692512178884, + "nauc_ndcg_at_5_std": 0.512281498494282, + "nauc_precision_at_1000_diff1": -0.33978313792118137, + "nauc_precision_at_1000_max": 0.2907342077224165, + "nauc_precision_at_1000_std": 0.3936720575758847, + "nauc_precision_at_100_diff1": -0.24660657473475445, + "nauc_precision_at_100_max": 0.3861235779172452, + "nauc_precision_at_100_std": 0.5986936361749609, + "nauc_precision_at_10_diff1": -0.11830469321718588, + "nauc_precision_at_10_max": 0.40771387510029194, + "nauc_precision_at_10_std": 0.5108579691233395, + "nauc_precision_at_1_diff1": 0.14735264735264755, + "nauc_precision_at_1_max": 0.31168831168831174, + "nauc_precision_at_1_std": 0.28791208791208733, + "nauc_precision_at_20_diff1": -0.05856766680606476, + "nauc_precision_at_20_max": 0.4804667215065963, + "nauc_precision_at_20_std": 0.5932454975345007, + "nauc_precision_at_3_diff1": -0.1743235944877092, + "nauc_precision_at_3_max": 0.30528406418230286, + "nauc_precision_at_3_std": 0.47497343159288485, + "nauc_precision_at_5_diff1": -0.029305361305361564, + "nauc_precision_at_5_max": 0.43590675990675964, + "nauc_precision_at_5_std": 0.5377529137529143, + "nauc_recall_at_1000_diff1": -0.03843212793046141, + "nauc_recall_at_1000_max": 0.47161521658470157, + "nauc_recall_at_1000_std": 0.6158365267254845, + "nauc_recall_at_100_diff1": 0.1326748396316691, + "nauc_recall_at_100_max": 0.39903392355230327, + "nauc_recall_at_100_std": 0.5841484488305276, + "nauc_recall_at_10_diff1": 0.19945575928636186, + "nauc_recall_at_10_max": 0.2974108076686036, + "nauc_recall_at_10_std": 0.3695192014253119, + "nauc_recall_at_1_diff1": 0.25234611831096293, + "nauc_recall_at_1_max": 0.19866732441765514, + "nauc_recall_at_1_std": 0.21652983886626678, + "nauc_recall_at_20_diff1": 0.24047442800151403, + "nauc_recall_at_20_max": 0.34503663200544493, + "nauc_recall_at_20_std": 0.4073793488699552, + "nauc_recall_at_3_diff1": 0.23114587093927097, + "nauc_recall_at_3_max": 0.2560474230250218, + "nauc_recall_at_3_std": 0.32609198226758757, + "nauc_recall_at_5_diff1": 0.27300098297834546, + "nauc_recall_at_5_max": 0.3066501918186581, + "nauc_recall_at_5_std": 0.35728065228196976, + "ndcg_at_1": 0.73, + "ndcg_at_10": 0.62314, + "ndcg_at_100": 0.4499, + "ndcg_at_1000": 0.41924, + "ndcg_at_20": 0.56517, + "ndcg_at_3": 0.71939, + "ndcg_at_5": 0.67405, + "precision_at_1": 0.8, + "precision_at_10": 0.652, + "precision_at_100": 0.4784, + "precision_at_1000": 0.198, + "precision_at_20": 0.598, + "precision_at_3": 0.78667, + "precision_at_5": 0.724, + "recall_at_1": 0.00199, + "recall_at_10": 0.01654, + "recall_at_100": 0.11214, + "recall_at_1000": 0.40574, + "recall_at_20": 0.03009, + "recall_at_3": 0.00588, + "recall_at_5": 0.00915 + } + ] + }, + "task_name": "TRECCOVID" +} \ No newline at end of file diff --git a/results/bm25s/0_1_10/Touche2020.json b/results/bm25s/0_1_10/Touche2020.json new file mode 100644 index 0000000000..aff6a91f71 --- /dev/null +++ b/results/bm25s/0_1_10/Touche2020.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f", + "evaluation_time": 101.41937065124512, + "kg_co2_emissions": null, + "mteb_version": "1.12.22", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.33053, + "map_at_1": 0.02771, + "map_at_10": 0.12511, + "map_at_100": 0.21104, + "map_at_1000": 0.22845, + "map_at_20": 0.16455, + "map_at_3": 0.05944, + "map_at_5": 0.08945, + "mrr_at_1": 0.42857142857142855, + "mrr_at_10": 0.5993197278911564, + "mrr_at_100": 0.6028628117913831, + "mrr_at_1000": 0.6028628117913831, + "mrr_at_20": 0.6019557823129251, + "mrr_at_3": 0.5578231292517006, + "mrr_at_5": 0.592517006802721, + "nauc_map_at_1000_diff1": 0.18049748554057368, + "nauc_map_at_1000_max": 0.0031061029826283345, + "nauc_map_at_1000_std": 0.034424212082991935, + "nauc_map_at_100_diff1": 0.17564275799558365, + "nauc_map_at_100_max": -0.0060326146504807235, + "nauc_map_at_100_std": 0.0186530487556058, + "nauc_map_at_10_diff1": 0.17832529803262248, + "nauc_map_at_10_max": 0.007635462573829391, + "nauc_map_at_10_std": -0.07015472254307786, + "nauc_map_at_1_diff1": 0.1738275951693309, + "nauc_map_at_1_max": -0.024646046709453563, + "nauc_map_at_1_std": -0.07198225982530083, + "nauc_map_at_20_diff1": 0.16688143939481234, + "nauc_map_at_20_max": 0.005142188930133397, + "nauc_map_at_20_std": -0.034560208428701035, + "nauc_map_at_3_diff1": 0.14898311472448825, + "nauc_map_at_3_max": -0.0710565254456865, + "nauc_map_at_3_std": -0.2072524341961518, + "nauc_map_at_5_diff1": 0.17075315818138237, + "nauc_map_at_5_max": -0.10630351150208865, + "nauc_map_at_5_std": -0.21478314685888364, + "nauc_mrr_at_1000_diff1": 0.15327723433931284, + "nauc_mrr_at_1000_max": 0.028922069602442615, + "nauc_mrr_at_1000_std": 0.020247267646860606, + "nauc_mrr_at_100_diff1": 0.15327723433931284, + "nauc_mrr_at_100_max": 0.028922069602442615, + "nauc_mrr_at_100_std": 0.020247267646860606, + "nauc_mrr_at_10_diff1": 0.15967181594191412, + "nauc_mrr_at_10_max": 0.03656765084275003, + "nauc_mrr_at_10_std": 0.028683521860730893, + "nauc_mrr_at_1_diff1": 0.13769738405636348, + "nauc_mrr_at_1_max": -0.017338061560474655, + "nauc_mrr_at_1_std": -0.024413762741842225, + "nauc_mrr_at_20_diff1": 0.15446418467071993, + "nauc_mrr_at_20_max": 0.03142490961753039, + "nauc_mrr_at_20_std": 0.023465516203141995, + "nauc_mrr_at_3_diff1": 0.12968881401200652, + "nauc_mrr_at_3_max": 0.022098944150334913, + "nauc_mrr_at_3_std": -0.012256701519098275, + "nauc_mrr_at_5_diff1": 0.1477226392359726, + "nauc_mrr_at_5_max": 0.016892204279189485, + "nauc_mrr_at_5_std": 0.009183857512629208, + "nauc_ndcg_at_1000_diff1": 0.24672553893711557, + "nauc_ndcg_at_1000_max": 0.0662891837826803, + "nauc_ndcg_at_1000_std": 0.1099076310460935, + "nauc_ndcg_at_100_diff1": 0.2710612972840078, + "nauc_ndcg_at_100_max": 0.017874779182773935, + "nauc_ndcg_at_100_std": 0.07808629142444955, + "nauc_ndcg_at_10_diff1": 0.18196102944722953, + "nauc_ndcg_at_10_max": 0.059808864830612325, + "nauc_ndcg_at_10_std": 0.0637071697862662, + "nauc_ndcg_at_1_diff1": 0.09292180184515422, + "nauc_ndcg_at_1_max": -0.05575090406621914, + "nauc_ndcg_at_1_std": -0.04480775075638091, + "nauc_ndcg_at_20_diff1": 0.21220945310565506, + "nauc_ndcg_at_20_max": 0.04535683899315996, + "nauc_ndcg_at_20_std": 0.027203773931377604, + "nauc_ndcg_at_3_diff1": 0.07926603108419188, + "nauc_ndcg_at_3_max": -0.06475566577613366, + "nauc_ndcg_at_3_std": -0.11673030538319452, + "nauc_ndcg_at_5_diff1": 0.11872198318810061, + "nauc_ndcg_at_5_max": -0.10252441923663748, + "nauc_ndcg_at_5_std": -0.11199368795756785, + "nauc_precision_at_1000_diff1": -0.1403763357341244, + "nauc_precision_at_1000_max": 0.13210643080061882, + "nauc_precision_at_1000_std": 0.21348613498051783, + "nauc_precision_at_100_diff1": 0.20138558711197135, + "nauc_precision_at_100_max": 0.10998617801638003, + "nauc_precision_at_100_std": 0.37474676579915933, + "nauc_precision_at_10_diff1": 0.1258707255001797, + "nauc_precision_at_10_max": 0.1730139572892476, + "nauc_precision_at_10_std": 0.27960318674916784, + "nauc_precision_at_1_diff1": 0.13769738405636348, + "nauc_precision_at_1_max": -0.017338061560474655, + "nauc_precision_at_1_std": -0.024413762741842225, + "nauc_precision_at_20_diff1": 0.13700215085934803, + "nauc_precision_at_20_max": 0.18285411329347256, + "nauc_precision_at_20_std": 0.31871481278917935, + "nauc_precision_at_3_diff1": 0.07215813720085541, + "nauc_precision_at_3_max": -0.0474903995704102, + "nauc_precision_at_3_std": -0.1476395432770433, + "nauc_precision_at_5_diff1": 0.09262348143951962, + "nauc_precision_at_5_max": -0.1342894108885341, + "nauc_precision_at_5_std": -0.12484210333486183, + "nauc_recall_at_1000_diff1": 0.331517586790325, + "nauc_recall_at_1000_max": 0.23698527433807529, + "nauc_recall_at_1000_std": 0.16450782266386302, + "nauc_recall_at_100_diff1": 0.335871459385677, + "nauc_recall_at_100_max": 0.06600050597513077, + "nauc_recall_at_100_std": 0.09220031677526468, + "nauc_recall_at_10_diff1": 0.21217185844433384, + "nauc_recall_at_10_max": 0.11359146058421925, + "nauc_recall_at_10_std": 0.03273845984539888, + "nauc_recall_at_1_diff1": 0.1738275951693309, + "nauc_recall_at_1_max": -0.024646046709453563, + "nauc_recall_at_1_std": -0.07198225982530083, + "nauc_recall_at_20_diff1": 0.22183136981243215, + "nauc_recall_at_20_max": 0.08655269132446805, + "nauc_recall_at_20_std": 0.023445677226348726, + "nauc_recall_at_3_diff1": 0.14528539860412976, + "nauc_recall_at_3_max": -0.029601227563150232, + "nauc_recall_at_3_std": -0.20092804185891472, + "nauc_recall_at_5_diff1": 0.18325728834640936, + "nauc_recall_at_5_max": -0.09478192263269947, + "nauc_recall_at_5_std": -0.20243760276028172, + "ndcg_at_1": 0.40816, + "ndcg_at_10": 0.33053, + "ndcg_at_100": 0.44971, + "ndcg_at_1000": 0.55486, + "ndcg_at_20": 0.34264, + "ndcg_at_3": 0.36529, + "ndcg_at_5": 0.36357, + "precision_at_1": 0.42857, + "precision_at_10": 0.30612, + "precision_at_100": 0.09367, + "precision_at_1000": 0.0162, + "precision_at_20": 0.24592, + "precision_at_3": 0.37415, + "precision_at_5": 0.37551, + "recall_at_1": 0.02771, + "recall_at_10": 0.2013, + "recall_at_100": 0.55575, + "recall_at_1000": 0.87444, + "recall_at_20": 0.31182, + "recall_at_3": 0.07463, + "recall_at_5": 0.12887 + } + ] + }, + "task_name": "Touche2020" +} \ No newline at end of file diff --git a/results/bm25s/0_1_10/model_meta.json b/results/bm25s/0_1_10/model_meta.json new file mode 100644 index 0000000000..3a22b72970 --- /dev/null +++ b/results/bm25s/0_1_10/model_meta.json @@ -0,0 +1 @@ +{"name": "bm25s", "revision": "0_1_10", "release_date": "2024-07-10", "languages": ["eng_Latn"], "n_parameters": null, "memory_usage": null, "max_tokens": null, "embed_dim": null, "license": null, "open_source": true, "similarity_fn_name": null, "framework": [], "loader": "BM25Search"} \ No newline at end of file