Skip to content

Commit

Permalink
Add Inf-retriever-v1 results (#90)
Browse files Browse the repository at this point in the history 
* Add C-MTEB retrieval results for inf-retriever-v1

* Update model_mete of inf-retriever-v1

---------

Co-authored-by: Samuel Yang <samuelyang150@gmail.com>
  • Loading branch information
@SamuelYang1
SamuelYang1 and SamuelYang1 authored Jan 16, 2025
1 parent c31c8e9 commit c33da34
Show file tree
Hide file tree
Showing 9 changed files with 1,265 additions and 0 deletions.
158 changes: 158 additions & 0 deletions ...lts/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/CmedqaRetrieval.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
{
"dataset_revision": "cd540c506dae1cf9e9a59c3e06f42030d54e7301",
"task_name": "CmedqaRetrieval",
"mteb_version": "1.28.0",
"scores": {
"dev": [
{
"ndcg_at_1": 0.3821,
"ndcg_at_3": 0.38438,
"ndcg_at_5": 0.40443,
"ndcg_at_10": 0.43306,
"ndcg_at_20": 0.46183,
"ndcg_at_100": 0.50646,
"ndcg_at_1000": 0.5282,
"map_at_1": 0.25153,
"map_at_3": 0.33038,
"map_at_5": 0.35228,
"map_at_10": 0.36924,
"map_at_20": 0.3796,
"map_at_100": 0.38792,
"map_at_1000": 0.38914,
"recall_at_1": 0.25153,
"recall_at_3": 0.38253,
"recall_at_5": 0.44508,
"recall_at_10": 0.53012,
"recall_at_20": 0.62915,
"recall_at_100": 0.83487,
"recall_at_1000": 0.98059,
"precision_at_1": 0.3821,
"precision_at_3": 0.21605,
"precision_at_5": 0.15664,
"precision_at_10": 0.09585,
"precision_at_20": 0.05731,
"precision_at_100": 0.01545,
"precision_at_1000": 0.00183,
"mrr_at_1": 0.382096,
"mrr_at_3": 0.436776,
"mrr_at_5": 0.449929,
"mrr_at_10": 0.460602,
"mrr_at_20": 0.466425,
"mrr_at_100": 0.470609,
"mrr_at_1000": 0.471081,
"nauc_ndcg_at_1_max": 0.472941,
"nauc_ndcg_at_1_std": 0.011506,
"nauc_ndcg_at_1_diff1": 0.65073,
"nauc_ndcg_at_3_max": 0.400306,
"nauc_ndcg_at_3_std": -0.021844,
"nauc_ndcg_at_3_diff1": 0.556355,
"nauc_ndcg_at_5_max": 0.384437,
"nauc_ndcg_at_5_std": -0.028594,
"nauc_ndcg_at_5_diff1": 0.543432,
"nauc_ndcg_at_10_max": 0.379058,
"nauc_ndcg_at_10_std": -0.022464,
"nauc_ndcg_at_10_diff1": 0.532127,
"nauc_ndcg_at_20_max": 0.386325,
"nauc_ndcg_at_20_std": -0.014661,
"nauc_ndcg_at_20_diff1": 0.530613,
"nauc_ndcg_at_100_max": 0.402745,
"nauc_ndcg_at_100_std": 0.012013,
"nauc_ndcg_at_100_diff1": 0.535367,
"nauc_ndcg_at_1000_max": 0.407732,
"nauc_ndcg_at_1000_std": 0.004336,
"nauc_ndcg_at_1000_diff1": 0.544837,
"nauc_map_at_1_max": 0.297667,
"nauc_map_at_1_std": -0.065036,
"nauc_map_at_1_diff1": 0.55916,
"nauc_map_at_3_max": 0.348091,
"nauc_map_at_3_std": -0.059474,
"nauc_map_at_3_diff1": 0.540836,
"nauc_map_at_5_max": 0.359765,
"nauc_map_at_5_std": -0.051715,
"nauc_map_at_5_diff1": 0.538996,
"nauc_map_at_10_max": 0.367189,
"nauc_map_at_10_std": -0.042277,
"nauc_map_at_10_diff1": 0.535705,
"nauc_map_at_20_max": 0.373278,
"nauc_map_at_20_std": -0.037319,
"nauc_map_at_20_diff1": 0.536623,
"nauc_map_at_100_max": 0.378037,
"nauc_map_at_100_std": -0.030759,
"nauc_map_at_100_diff1": 0.538127,
"nauc_map_at_1000_max": 0.378518,
"nauc_map_at_1000_std": -0.030761,
"nauc_map_at_1000_diff1": 0.538489,
"nauc_recall_at_1_max": 0.297667,
"nauc_recall_at_1_std": -0.065036,
"nauc_recall_at_1_diff1": 0.55916,
"nauc_recall_at_3_max": 0.302726,
"nauc_recall_at_3_std": -0.064455,
"nauc_recall_at_3_diff1": 0.47646,
"nauc_recall_at_5_max": 0.295304,
"nauc_recall_at_5_std": -0.053595,
"nauc_recall_at_5_diff1": 0.446824,
"nauc_recall_at_10_max": 0.280634,
"nauc_recall_at_10_std": -0.02122,
"nauc_recall_at_10_diff1": 0.40478,
"nauc_recall_at_20_max": 0.278269,
"nauc_recall_at_20_std": -0.000429,
"nauc_recall_at_20_diff1": 0.372018,
"nauc_recall_at_100_max": 0.30131,
"nauc_recall_at_100_std": 0.193349,
"nauc_recall_at_100_diff1": 0.292745,
"nauc_recall_at_1000_max": 0.33964,
"nauc_recall_at_1000_std": 0.532129,
"nauc_recall_at_1000_diff1": 0.197514,
"nauc_precision_at_1_max": 0.472941,
"nauc_precision_at_1_std": 0.011506,
"nauc_precision_at_1_diff1": 0.65073,
"nauc_precision_at_3_max": 0.454966,
"nauc_precision_at_3_std": 0.05432,
"nauc_precision_at_3_diff1": 0.440588,
"nauc_precision_at_5_max": 0.439257,
"nauc_precision_at_5_std": 0.081048,
"nauc_precision_at_5_diff1": 0.375711,
"nauc_precision_at_10_max": 0.412745,
"nauc_precision_at_10_std": 0.120234,
"nauc_precision_at_10_diff1": 0.299403,
"nauc_precision_at_20_max": 0.398018,
"nauc_precision_at_20_std": 0.152937,
"nauc_precision_at_20_diff1": 0.244872,
"nauc_precision_at_100_max": 0.35012,
"nauc_precision_at_100_std": 0.240025,
"nauc_precision_at_100_diff1": 0.132948,
"nauc_precision_at_1000_max": 0.291727,
"nauc_precision_at_1000_std": 0.206349,
"nauc_precision_at_1000_diff1": 0.0646,
"nauc_mrr_at_1_max": 0.472941,
"nauc_mrr_at_1_std": 0.011506,
"nauc_mrr_at_1_diff1": 0.65073,
"nauc_mrr_at_3_max": 0.456503,
"nauc_mrr_at_3_std": 0.005387,
"nauc_mrr_at_3_diff1": 0.618489,
"nauc_mrr_at_5_max": 0.453089,
"nauc_mrr_at_5_std": 0.007656,
"nauc_mrr_at_5_diff1": 0.61148,
"nauc_mrr_at_10_max": 0.452298,
"nauc_mrr_at_10_std": 0.0115,
"nauc_mrr_at_10_diff1": 0.607758,
"nauc_mrr_at_20_max": 0.452277,
"nauc_mrr_at_20_std": 0.011928,
"nauc_mrr_at_20_diff1": 0.606927,
"nauc_mrr_at_100_max": 0.453238,
"nauc_mrr_at_100_std": 0.013097,
"nauc_mrr_at_100_diff1": 0.607616,
"nauc_mrr_at_1000_max": 0.453298,
"nauc_mrr_at_1000_std": 0.012893,
"nauc_mrr_at_1000_diff1": 0.607901,
"main_score": 0.43306,
"hf_subset": "default",
"languages": [
"cmn-Hans"
]
}
]
},
"evaluation_time": 16793.309911966324,
"kg_co2_emissions": null
}
158 changes: 158 additions & 0 deletions results/infly__inf-retriever-v1/d2d074546028c0012b5cc6af78c4fac24896e67f/CovidRetrieval.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
{
"dataset_revision": "1271c7809071a13532e05f25fb53511ffce77117",
"task_name": "CovidRetrieval",
"mteb_version": "1.28.0",
"scores": {
"dev": [
{
"ndcg_at_1": 0.77977,
"ndcg_at_3": 0.84489,
"ndcg_at_5": 0.86183,
"ndcg_at_10": 0.87159,
"ndcg_at_20": 0.87638,
"ndcg_at_100": 0.87976,
"ndcg_at_1000": 0.88026,
"map_at_1": 0.77713,
"map_at_3": 0.82947,
"map_at_5": 0.83909,
"map_at_10": 0.84332,
"map_at_20": 0.84472,
"map_at_100": 0.8452,
"map_at_1000": 0.84522,
"recall_at_1": 0.77713,
"recall_at_3": 0.88857,
"recall_at_5": 0.92914,
"recall_at_10": 0.95838,
"recall_at_20": 0.97682,
"recall_at_100": 0.99473,
"recall_at_1000": 0.99895,
"precision_at_1": 0.77977,
"precision_at_3": 0.29856,
"precision_at_5": 0.18757,
"precision_at_10": 0.09684,
"precision_at_20": 0.04937,
"precision_at_100": 0.01005,
"precision_at_1000": 0.00101,
"mrr_at_1": 0.779768,
"mrr_at_3": 0.82947,
"mrr_at_5": 0.839322,
"mrr_at_10": 0.843297,
"mrr_at_20": 0.844602,
"mrr_at_100": 0.845085,
"mrr_at_1000": 0.845098,
"nauc_ndcg_at_1_max": 0.106257,
"nauc_ndcg_at_1_std": -0.37587,
"nauc_ndcg_at_1_diff1": 0.876739,
"nauc_ndcg_at_3_max": 0.114731,
"nauc_ndcg_at_3_std": -0.394782,
"nauc_ndcg_at_3_diff1": 0.851518,
"nauc_ndcg_at_5_max": 0.133553,
"nauc_ndcg_at_5_std": -0.380635,
"nauc_ndcg_at_5_diff1": 0.847753,
"nauc_ndcg_at_10_max": 0.135939,
"nauc_ndcg_at_10_std": -0.353862,
"nauc_ndcg_at_10_diff1": 0.852025,
"nauc_ndcg_at_20_max": 0.123043,
"nauc_ndcg_at_20_std": -0.344532,
"nauc_ndcg_at_20_diff1": 0.854869,
"nauc_ndcg_at_100_max": 0.12359,
"nauc_ndcg_at_100_std": -0.362481,
"nauc_ndcg_at_100_diff1": 0.856223,
"nauc_ndcg_at_1000_max": 0.12554,
"nauc_ndcg_at_1000_std": -0.364419,
"nauc_ndcg_at_1000_diff1": 0.856547,
"nauc_map_at_1_max": 0.107918,
"nauc_map_at_1_std": -0.380017,
"nauc_map_at_1_diff1": 0.88115,
"nauc_map_at_3_max": 0.112948,
"nauc_map_at_3_std": -0.387444,
"nauc_map_at_3_diff1": 0.859009,
"nauc_map_at_5_max": 0.123632,
"nauc_map_at_5_std": -0.381401,
"nauc_map_at_5_diff1": 0.857334,
"nauc_map_at_10_max": 0.125679,
"nauc_map_at_10_std": -0.37196,
"nauc_map_at_10_diff1": 0.858887,
"nauc_map_at_20_max": 0.122576,
"nauc_map_at_20_std": -0.369194,
"nauc_map_at_20_diff1": 0.859492,
"nauc_map_at_100_max": 0.123137,
"nauc_map_at_100_std": -0.371156,
"nauc_map_at_100_diff1": 0.859659,
"nauc_map_at_1000_max": 0.123179,
"nauc_map_at_1000_std": -0.371229,
"nauc_map_at_1000_diff1": 0.859666,
"nauc_recall_at_1_max": 0.107918,
"nauc_recall_at_1_std": -0.380017,
"nauc_recall_at_1_diff1": 0.88115,
"nauc_recall_at_3_max": 0.115712,
"nauc_recall_at_3_std": -0.420403,
"nauc_recall_at_3_diff1": 0.822085,
"nauc_recall_at_5_max": 0.193703,
"nauc_recall_at_5_std": -0.364651,
"nauc_recall_at_5_diff1": 0.790229,
"nauc_recall_at_10_max": 0.247021,
"nauc_recall_at_10_std": -0.130473,
"nauc_recall_at_10_diff1": 0.790931,
"nauc_recall_at_20_max": 0.083776,
"nauc_recall_at_20_std": 0.195544,
"nauc_recall_at_20_diff1": 0.801544,
"nauc_recall_at_100_max": -0.164048,
"nauc_recall_at_100_std": -0.04217,
"nauc_recall_at_100_diff1": 0.773525,
"nauc_recall_at_1000_max": 0.357809,
"nauc_recall_at_1000_std": -0.171046,
"nauc_recall_at_1000_diff1": 0.722168,
"nauc_precision_at_1_max": 0.106257,
"nauc_precision_at_1_std": -0.37587,
"nauc_precision_at_1_diff1": 0.876739,
"nauc_precision_at_3_max": 0.123466,
"nauc_precision_at_3_std": -0.352022,
"nauc_precision_at_3_diff1": 0.693658,
"nauc_precision_at_5_max": 0.218156,
"nauc_precision_at_5_std": -0.267725,
"nauc_precision_at_5_diff1": 0.556505,
"nauc_precision_at_10_max": 0.280424,
"nauc_precision_at_10_std": -0.039005,
"nauc_precision_at_10_diff1": 0.400286,
"nauc_precision_at_20_max": 0.178965,
"nauc_precision_at_20_std": 0.200597,
"nauc_precision_at_20_diff1": 0.197173,
"nauc_precision_at_100_max": 0.195903,
"nauc_precision_at_100_std": 0.151328,
"nauc_precision_at_100_diff1": -0.268167,
"nauc_precision_at_1000_max": 0.302629,
"nauc_precision_at_1000_std": 0.186426,
"nauc_precision_at_1000_diff1": -0.49766,
"nauc_mrr_at_1_max": 0.114448,
"nauc_mrr_at_1_std": -0.384039,
"nauc_mrr_at_1_diff1": 0.876739,
"nauc_mrr_at_3_max": 0.120501,
"nauc_mrr_at_3_std": -0.396139,
"nauc_mrr_at_3_diff1": 0.858329,
"nauc_mrr_at_5_max": 0.125492,
"nauc_mrr_at_5_std": -0.386156,
"nauc_mrr_at_5_diff1": 0.856892,
"nauc_mrr_at_10_max": 0.126474,
"nauc_mrr_at_10_std": -0.377216,
"nauc_mrr_at_10_diff1": 0.858803,
"nauc_mrr_at_20_max": 0.123774,
"nauc_mrr_at_20_std": -0.375013,
"nauc_mrr_at_20_diff1": 0.8595,
"nauc_mrr_at_100_max": 0.124338,
"nauc_mrr_at_100_std": -0.376993,
"nauc_mrr_at_100_diff1": 0.859667,
"nauc_mrr_at_1000_max": 0.124381,
"nauc_mrr_at_1000_std": -0.377067,
"nauc_mrr_at_1000_diff1": 0.859674,
"main_score": 0.87159,
"hf_subset": "default",
"languages": [
"cmn-Hans"
]
}
]
},
"evaluation_time": 17476.262266874313,
"kg_co2_emissions": null
}
Loading

0 comments on commit c33da34

Please sign in to comment.