Skip to content

Commit

Permalink
Add FaMTEB (Farsi/Persian Text Embedding Benchmark) (#92)
Browse files Browse the repository at this point in the history 
* Add FaMTEB results for new models

* Add FaMTEB results for existing models

* Modify results.py and paths.json

* Fix bugs to pass tests

* Fix mteb version

* Resolve conflicts

* Update paths.json

* fix conflict

---------

Co-authored-by: mehran <mehan.sarmadi16@gmail.com>
  • Loading branch information
@mehran-sarmadi
mehran-sarmadi and mehran authored Feb 1, 2025
1 parent dbd0d75 commit 38fafcb
Show file tree
Hide file tree
Showing 1,261 changed files with 176,752 additions and 48,194 deletions.
97,366 changes: 49,313 additions & 48,053 deletions paths.json
View file

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions results.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@
"est-eng",
"eus-eng",
"fa",
"fas-Arab",
"fao-eng",
"fi",
"fin-eng",
Expand Down Expand Up @@ -240,6 +241,7 @@
"LEMBSummScreenFDRetrieval",
"MSMARCO",
"MSMARCO-PL",
"MSMARCO-Fa",
"MultilingualSentiment",
"Ocnli",
"TNews",
Expand All @@ -255,6 +257,7 @@
"MMarcoRetrieval",
"MSMARCO",
"MSMARCO-PL",
"MSMARCO-Fa",
"T2Reranking",
"T2Retrieval",
"VideoRetrieval",
Expand Down
158 changes: 158 additions & 0 deletions ...ibaba-NLP__gte-multilingual-base/7fc06782350c1a83f88b15dd4b38ef853d3b8503/ArguAna-Fa.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
{
"dataset_revision": "bd449a14a4a7d5644ffd380a957c3c4bab83bf50",
"task_name": "ArguAna-Fa",
"mteb_version": "1.25.8",
"scores": {
"test": [
{
"ndcg_at_1": 0.25889,
"ndcg_at_3": 0.40073,
"ndcg_at_5": 0.45182,
"ndcg_at_10": 0.50397,
"ndcg_at_20": 0.53162,
"ndcg_at_100": 0.54813,
"ndcg_at_1000": 0.54977,
"map_at_1": 0.25889,
"map_at_3": 0.36463,
"map_at_5": 0.39322,
"map_at_10": 0.41484,
"map_at_20": 0.42251,
"map_at_100": 0.42505,
"map_at_1000": 0.42511,
"recall_at_1": 0.25889,
"recall_at_3": 0.50569,
"recall_at_5": 0.62873,
"recall_at_10": 0.78947,
"recall_at_20": 0.89829,
"recall_at_100": 0.98364,
"recall_at_1000": 0.99644,
"precision_at_1": 0.25889,
"precision_at_3": 0.16856,
"precision_at_5": 0.12575,
"precision_at_10": 0.07895,
"precision_at_20": 0.04491,
"precision_at_100": 0.00984,
"precision_at_1000": 0.001,
"mrr_at_1": 0.263158,
"mrr_at_3": 0.366761,
"mrr_at_5": 0.395033,
"mrr_at_10": 0.416376,
"mrr_at_20": 0.424144,
"mrr_at_100": 0.426716,
"mrr_at_1000": 0.42678,
"nauc_ndcg_at_1_max": 0.008706,
"nauc_ndcg_at_1_std": -0.035326,
"nauc_ndcg_at_1_diff1": 0.130705,
"nauc_ndcg_at_3_max": 0.031295,
"nauc_ndcg_at_3_std": -0.032176,
"nauc_ndcg_at_3_diff1": 0.079723,
"nauc_ndcg_at_5_max": 0.031441,
"nauc_ndcg_at_5_std": -0.02984,
"nauc_ndcg_at_5_diff1": 0.070577,
"nauc_ndcg_at_10_max": 0.040205,
"nauc_ndcg_at_10_std": -0.026616,
"nauc_ndcg_at_10_diff1": 0.070194,
"nauc_ndcg_at_20_max": 0.047201,
"nauc_ndcg_at_20_std": -0.022678,
"nauc_ndcg_at_20_diff1": 0.080201,
"nauc_ndcg_at_100_max": 0.040574,
"nauc_ndcg_at_100_std": -0.018592,
"nauc_ndcg_at_100_diff1": 0.08571,
"nauc_ndcg_at_1000_max": 0.036597,
"nauc_ndcg_at_1000_std": -0.022744,
"nauc_ndcg_at_1000_diff1": 0.084229,
"nauc_map_at_1_max": 0.008706,
"nauc_map_at_1_std": -0.035326,
"nauc_map_at_1_diff1": 0.130705,
"nauc_map_at_3_max": 0.025991,
"nauc_map_at_3_std": -0.032148,
"nauc_map_at_3_diff1": 0.089708,
"nauc_map_at_5_max": 0.025958,
"nauc_map_at_5_std": -0.030882,
"nauc_map_at_5_diff1": 0.084998,
"nauc_map_at_10_max": 0.029082,
"nauc_map_at_10_std": -0.029939,
"nauc_map_at_10_diff1": 0.08539,
"nauc_map_at_20_max": 0.030962,
"nauc_map_at_20_std": -0.028818,
"nauc_map_at_20_diff1": 0.088222,
"nauc_map_at_100_max": 0.030192,
"nauc_map_at_100_std": -0.02808,
"nauc_map_at_100_diff1": 0.089145,
"nauc_map_at_1000_max": 0.030049,
"nauc_map_at_1000_std": -0.028203,
"nauc_map_at_1000_diff1": 0.089096,
"nauc_recall_at_1_max": 0.008706,
"nauc_recall_at_1_std": -0.035326,
"nauc_recall_at_1_diff1": 0.130705,
"nauc_recall_at_3_max": 0.046177,
"nauc_recall_at_3_std": -0.032556,
"nauc_recall_at_3_diff1": 0.052486,
"nauc_recall_at_5_max": 0.048704,
"nauc_recall_at_5_std": -0.026508,
"nauc_recall_at_5_diff1": 0.024514,
"nauc_recall_at_10_max": 0.097601,
"nauc_recall_at_10_std": -0.008275,
"nauc_recall_at_10_diff1": -0.000901,
"nauc_recall_at_20_max": 0.211201,
"nauc_recall_at_20_std": 0.042908,
"nauc_recall_at_20_diff1": 0.033366,
"nauc_recall_at_100_max": 0.477208,
"nauc_recall_at_100_std": 0.615253,
"nauc_recall_at_100_diff1": 0.160113,
"nauc_recall_at_1000_max": -0.021532,
"nauc_recall_at_1000_std": 0.627524,
"nauc_recall_at_1000_diff1": -0.371153,
"nauc_precision_at_1_max": 0.008706,
"nauc_precision_at_1_std": -0.035326,
"nauc_precision_at_1_diff1": 0.130705,
"nauc_precision_at_3_max": 0.046177,
"nauc_precision_at_3_std": -0.032556,
"nauc_precision_at_3_diff1": 0.052486,
"nauc_precision_at_5_max": 0.048704,
"nauc_precision_at_5_std": -0.026508,
"nauc_precision_at_5_diff1": 0.024514,
"nauc_precision_at_10_max": 0.097601,
"nauc_precision_at_10_std": -0.008275,
"nauc_precision_at_10_diff1": -0.000901,
"nauc_precision_at_20_max": 0.211201,
"nauc_precision_at_20_std": 0.042908,
"nauc_precision_at_20_diff1": 0.033366,
"nauc_precision_at_100_max": 0.477208,
"nauc_precision_at_100_std": 0.615253,
"nauc_precision_at_100_diff1": 0.160113,
"nauc_precision_at_1000_max": -0.021532,
"nauc_precision_at_1000_std": 0.627524,
"nauc_precision_at_1000_diff1": -0.371153,
"nauc_mrr_at_1_max": 0.019055,
"nauc_mrr_at_1_std": -0.0341,
"nauc_mrr_at_1_diff1": 0.116596,
"nauc_mrr_at_3_max": 0.022974,
"nauc_mrr_at_3_std": -0.031969,
"nauc_mrr_at_3_diff1": 0.07646,
"nauc_mrr_at_5_max": 0.024801,
"nauc_mrr_at_5_std": -0.030518,
"nauc_mrr_at_5_diff1": 0.071892,
"nauc_mrr_at_10_max": 0.02759,
"nauc_mrr_at_10_std": -0.03,
"nauc_mrr_at_10_diff1": 0.071739,
"nauc_mrr_at_20_max": 0.029798,
"nauc_mrr_at_20_std": -0.028892,
"nauc_mrr_at_20_diff1": 0.07478,
"nauc_mrr_at_100_max": 0.0292,
"nauc_mrr_at_100_std": -0.028223,
"nauc_mrr_at_100_diff1": 0.075768,
"nauc_mrr_at_1000_max": 0.029057,
"nauc_mrr_at_1000_std": -0.028346,
"nauc_mrr_at_1000_diff1": 0.075717,
"main_score": 0.50397,
"hf_subset": "default",
"languages": [
"fas-Arab"
]
}
]
},
"evaluation_time": 46.31877279281616,
"kg_co2_emissions": null
}
34 changes: 34 additions & 0 deletions ...P__gte-multilingual-base/7fc06782350c1a83f88b15dd4b38ef853d3b8503/BeytooteClustering.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
{
"dataset_revision": "62ca5aecb9414214162569f2f1bfb07aa219a70e",
"task_name": "BeytooteClustering",
"mteb_version": "1.25.8",
"scores": {
"test": [
{
"v_measures": {
"Level 0": [
0.549041,
0.615929,
0.640994,
0.660619,
0.654895,
0.627701,
0.617975,
0.636832,
0.58294,
0.665533
]
},
"v_measure": 0.625246,
"v_measure_std": 0.034465,
"main_score": 0.625246,
"hf_subset": "default",
"languages": [
"fas-Arab"
]
}
]
},
"evaluation_time": 118.68936443328857,
"kg_co2_emissions": null
}
58 changes: 58 additions & 0 deletions .../Alibaba-NLP__gte-multilingual-base/7fc06782350c1a83f88b15dd4b38ef853d3b8503/CExaPPC.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
{
"dataset_revision": "68a0ff474739367a36c8066ee04802a65aefc117",
"task_name": "CExaPPC",
"mteb_version": "1.25.8",
"scores": {
"test": [
{
"similarity_accuracy": 0.949126,
"similarity_accuracy_threshold": 0.650263,
"similarity_f1": 0.936063,
"similarity_f1_threshold": 0.649996,
"similarity_precision": 0.927346,
"similarity_recall": 0.944946,
"similarity_ap": 0.984153,
"cosine_accuracy": 0.949126,
"cosine_accuracy_threshold": 0.650263,
"cosine_f1": 0.936063,
"cosine_f1_threshold": 0.649996,
"cosine_precision": 0.927346,
"cosine_recall": 0.944946,
"cosine_ap": 0.984153,
"manhattan_accuracy": 0.947645,
"manhattan_accuracy_threshold": 18.084282,
"manhattan_f1": 0.934722,
"manhattan_f1_threshold": 18.185123,
"manhattan_precision": 0.917632,
"manhattan_recall": 0.952461,
"manhattan_ap": 0.983664,
"euclidean_accuracy": 0.949126,
"euclidean_accuracy_threshold": 0.836346,
"euclidean_f1": 0.936063,
"euclidean_f1_threshold": 0.836665,
"euclidean_precision": 0.927346,
"euclidean_recall": 0.944946,
"euclidean_ap": 0.984153,
"dot_accuracy": 0.949126,
"dot_accuracy_threshold": 0.650263,
"dot_f1": 0.936063,
"dot_f1_threshold": 0.649996,
"dot_precision": 0.927346,
"dot_recall": 0.944946,
"dot_ap": 0.984153,
"max_accuracy": 0.949126,
"max_f1": 0.936063,
"max_precision": 0.927346,
"max_recall": 0.952461,
"max_ap": 0.984153,
"main_score": 0.984153,
"hf_subset": "default",
"languages": [
"fas-Arab"
]
}
]
},
"evaluation_time": 20.422319412231445,
"kg_co2_emissions": null
}
Loading

0 comments on commit 38fafcb

Please sign in to comment.