diff --git a/docs/mmteb/points/927.jsonl b/docs/mmteb/points/927.jsonl new file mode 100644 index 0000000000..64ea007b58 --- /dev/null +++ b/docs/mmteb/points/927.jsonl @@ -0,0 +1,2 @@ +{"GitHub": "gentaiscool", "New dataset": 18} +{"GitHub": "KennethEnevoldsen", "Review PR": 2} \ No newline at end of file diff --git a/mteb/tasks/Classification/__init__.py b/mteb/tasks/Classification/__init__.py index 89826de5cb..9559feeb80 100644 --- a/mteb/tasks/Classification/__init__.py +++ b/mteb/tasks/Classification/__init__.py @@ -86,6 +86,7 @@ from .multilingual.MultilingualSentimentClassification import * from .multilingual.NaijaSenti import * from .multilingual.NordicLangClassification import * +from .multilingual.NusaParagraphTopicClassification import * from .multilingual.NusaXSenti import * from .multilingual.ScalaClassification import * from .multilingual.SIB200Classification import * diff --git a/mteb/tasks/Classification/multilingual/NusaParagraphTopicClassification.py b/mteb/tasks/Classification/multilingual/NusaParagraphTopicClassification.py new file mode 100644 index 0000000000..c7d4900de3 --- /dev/null +++ b/mteb/tasks/Classification/multilingual/NusaParagraphTopicClassification.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from mteb.abstasks.TaskMetadata import TaskMetadata + +from ....abstasks import AbsTaskClassification, MultilingualTask + +_LANGUAGES = { + "btk": ["bbc-Latn"], + "bew": ["bew-Latn"], + "bug": ["bug-Latn"], + "jav": ["jav-Latn"], + "mad": ["mad-Latn"], + "mak": ["mak-Latn"], + "min": ["min-Latn"], + "mui": ["mui-Latn"], + "rej": ["rej-Latn"], + "sun": ["sun-Latn"], +} + + +class NusaParagraphTopicClassification(MultilingualTask, AbsTaskClassification): + metadata = TaskMetadata( + name="NusaParagraphTopicClassification", + dataset={ + "path": "gentaiscool/nusaparagraph_topic", + "revision": "abb43f8d5b9510b8724b48283aca26c4733eac5d", + }, + description="NusaParagraphTopicClassification is a multi-class topic classification on 10 Indonesian languages.", + reference="https://github.com/IndoNLP/nusa-writes", + category="s2s", + type="Classification", + eval_splits=["test"], + eval_langs=_LANGUAGES, + main_score="f1", + date=("2021-08-01", "2022-07-01"), + form=["written"], + domains=["Non-fiction", "Fiction"], + task_subtypes=["Topic classification"], + license="Apache 2.0", + socioeconomic_status="mixed", + annotations_creators="human-annotated", + dialect=[], + text_creation="found", + bibtex_citation=""" + @inproceedings{cahyawijaya-etal-2023-nusawrites, + title = "NusaWrites: Constructing High-Quality Corpora for Underrepresented and Extremely Low-Resource Languages", + author = "Cahyawijaya, Samuel and Lovenia, Holy and Koto, Fajri and Adhista, Dea and Dave, Emmanuel and Oktavianti, Sarah and Akbar, Salsabil and Lee, Jhonson and Shadieq, Nuur and Cenggoro, Tjeng Wawan and Linuwih, Hanung and Wilie, Bryan and Muridan, Galih and Winata, Genta and Moeljadi, David and Aji, Alham Fikri and Purwarianti, Ayu and Fung, Pascale", + editor = "Park, Jong C. and Arase, Yuki and Hu, Baotian and Lu, Wei and Wijaya, Derry and Purwarianti, Ayu and Krisnadhi, Adila Alfa", + booktitle = "Proceedings of the 13th International Joint Conference on Natural Language Processing and the 3rd Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)", + month = nov, + year = "2023", + address = "Nusa Dua, Bali", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2023.ijcnlp-main.60", + pages = "921--945", + } + """, + n_samples={"train": 15516, "validation": 2948, "test": 6250}, + avg_character_length={"train": 740.24, "validation": 740.66, "test": 740.71}, + ) diff --git a/results/intfloat__multilingual-e5-small/0a68dcd3dad5b4962a78daa930087728292b241d/NusaParagraphTopicClassification.json b/results/intfloat__multilingual-e5-small/0a68dcd3dad5b4962a78daa930087728292b241d/NusaParagraphTopicClassification.json new file mode 100644 index 0000000000..40e6be2592 --- /dev/null +++ b/results/intfloat__multilingual-e5-small/0a68dcd3dad5b4962a78daa930087728292b241d/NusaParagraphTopicClassification.json @@ -0,0 +1,631 @@ +{ + "dataset_revision": "abb43f8d5b9510b8724b48283aca26c4733eac5d", + "evaluation_time": 31.910111904144287, + "kg_co2_emissions": null, + "mteb_version": "1.12.30", + "scores": { + "test": [ + { + "accuracy": 0.6334, + "f1": 0.6171183470608116, + "f1_weighted": 0.629464721103177, + "hf_subset": "btk", + "languages": [ + "bbc-Latn" + ], + "main_score": 0.6171183470608116, + "scores_per_experiment": [ + { + "accuracy": 0.59, + "f1": 0.5852878404051078, + "f1_weighted": 0.5958927453217002 + }, + { + "accuracy": 0.62, + "f1": 0.600587569376949, + "f1_weighted": 0.6015903595912627 + }, + { + "accuracy": 0.696, + "f1": 0.6760889882154619, + "f1_weighted": 0.6924571272692137 + }, + { + "accuracy": 0.64, + "f1": 0.6173446272666636, + "f1_weighted": 0.6327655573975481 + }, + { + "accuracy": 0.624, + "f1": 0.5986244170348578, + "f1_weighted": 0.6074905539687783 + }, + { + "accuracy": 0.658, + "f1": 0.6358373683853095, + "f1_weighted": 0.6509800878050068 + }, + { + "accuracy": 0.572, + "f1": 0.5690495790315273, + "f1_weighted": 0.581213494781202 + }, + { + "accuracy": 0.65, + "f1": 0.6304842998267575, + "f1_weighted": 0.6496684375932065 + }, + { + "accuracy": 0.644, + "f1": 0.6299587015672454, + "f1_weighted": 0.6502681630348887 + }, + { + "accuracy": 0.64, + "f1": 0.627920079498236, + "f1_weighted": 0.6323206842689628 + } + ] + }, + { + "accuracy": 0.68675, + "f1": 0.6678745690209558, + "f1_weighted": 0.6878928377540281, + "hf_subset": "bew", + "languages": [ + "bew-Latn" + ], + "main_score": 0.6678745690209558, + "scores_per_experiment": [ + { + "accuracy": 0.64625, + "f1": 0.6322727297666426, + "f1_weighted": 0.6592130253059881 + }, + { + "accuracy": 0.69125, + "f1": 0.663240860460134, + "f1_weighted": 0.6932953236033225 + }, + { + "accuracy": 0.66125, + "f1": 0.6531668710206416, + "f1_weighted": 0.6754609390663449 + }, + { + "accuracy": 0.695, + "f1": 0.6758684252299783, + "f1_weighted": 0.6964981076543918 + }, + { + "accuracy": 0.69, + "f1": 0.651473632085287, + "f1_weighted": 0.6754998687407036 + }, + { + "accuracy": 0.73125, + "f1": 0.7094399385343673, + "f1_weighted": 0.7314875092958686 + }, + { + "accuracy": 0.72875, + "f1": 0.7150086094859092, + "f1_weighted": 0.7296549282637496 + }, + { + "accuracy": 0.655, + "f1": 0.6451372316903234, + "f1_weighted": 0.6600691070006784 + }, + { + "accuracy": 0.65125, + "f1": 0.6358222537320755, + "f1_weighted": 0.6402935294276272 + }, + { + "accuracy": 0.7175, + "f1": 0.6973151382041993, + "f1_weighted": 0.7174560391816064 + } + ] + }, + { + "accuracy": 0.6503333333333334, + "f1": 0.44533490659642155, + "f1_weighted": 0.5778813318278211, + "hf_subset": "bug", + "languages": [ + "bug-Latn" + ], + "main_score": 0.44533490659642155, + "scores_per_experiment": [ + { + "accuracy": 0.61, + "f1": 0.4114791266678888, + "f1_weighted": 0.5414099616118933 + }, + { + "accuracy": 0.6433333333333333, + "f1": 0.4392030792840382, + "f1_weighted": 0.5696605604687246 + }, + { + "accuracy": 0.6766666666666666, + "f1": 0.4697495627302843, + "f1_weighted": 0.6069091885768013 + }, + { + "accuracy": 0.6466666666666666, + "f1": 0.44455962019758105, + "f1_weighted": 0.5736951562727561 + }, + { + "accuracy": 0.64, + "f1": 0.44168181982282356, + "f1_weighted": 0.5717484945947696 + }, + { + "accuracy": 0.66, + "f1": 0.4545023388773389, + "f1_weighted": 0.5880218872718873 + }, + { + "accuracy": 0.6466666666666666, + "f1": 0.4338039458102571, + "f1_weighted": 0.566220688043727 + }, + { + "accuracy": 0.6733333333333333, + "f1": 0.4684968853082196, + "f1_weighted": 0.6027469971019468 + }, + { + "accuracy": 0.65, + "f1": 0.4399980836214573, + "f1_weighted": 0.5740811958220885 + }, + { + "accuracy": 0.6566666666666666, + "f1": 0.4498746036443266, + "f1_weighted": 0.5843191885136169 + } + ] + }, + { + "accuracy": 0.68075, + "f1": 0.660155025408092, + "f1_weighted": 0.6759930020418418, + "hf_subset": "jav", + "languages": [ + "jav-Latn" + ], + "main_score": 0.660155025408092, + "scores_per_experiment": [ + { + "accuracy": 0.70375, + "f1": 0.6847649326728814, + "f1_weighted": 0.7077673546461796 + }, + { + "accuracy": 0.69125, + "f1": 0.6686803719870673, + "f1_weighted": 0.6880433606719385 + }, + { + "accuracy": 0.7, + "f1": 0.6784226413470056, + "f1_weighted": 0.6869195249472552 + }, + { + "accuracy": 0.6575, + "f1": 0.6350688729596057, + "f1_weighted": 0.6555931333347271 + }, + { + "accuracy": 0.62875, + "f1": 0.59445380923915, + "f1_weighted": 0.6117551600143775 + }, + { + "accuracy": 0.66875, + "f1": 0.6527602624386722, + "f1_weighted": 0.6653199559754615 + }, + { + "accuracy": 0.70625, + "f1": 0.6883303308128185, + "f1_weighted": 0.7058423615373113 + }, + { + "accuracy": 0.67125, + "f1": 0.6510035114005515, + "f1_weighted": 0.6642991582996877 + }, + { + "accuracy": 0.69625, + "f1": 0.6805498011412016, + "f1_weighted": 0.687553555196325 + }, + { + "accuracy": 0.68375, + "f1": 0.6675157200819667, + "f1_weighted": 0.6868364557951552 + } + ] + }, + { + "accuracy": 0.638, + "f1": 0.5995558948750676, + "f1_weighted": 0.6412866525067649, + "hf_subset": "mad", + "languages": [ + "mad-Latn" + ], + "main_score": 0.5995558948750676, + "scores_per_experiment": [ + { + "accuracy": 0.5742857142857143, + "f1": 0.5035475665060216, + "f1_weighted": 0.5649910543100422 + }, + { + "accuracy": 0.6442857142857142, + "f1": 0.6004080345307737, + "f1_weighted": 0.6551125552976121 + }, + { + "accuracy": 0.6457142857142857, + "f1": 0.6040474022035318, + "f1_weighted": 0.635934183231404 + }, + { + "accuracy": 0.6485714285714286, + "f1": 0.631406759176383, + "f1_weighted": 0.6548543594697213 + }, + { + "accuracy": 0.6471428571428571, + "f1": 0.6191920933012576, + "f1_weighted": 0.6581171550936497 + }, + { + "accuracy": 0.6428571428571429, + "f1": 0.6073610653229178, + "f1_weighted": 0.6452590930035225 + }, + { + "accuracy": 0.6285714285714286, + "f1": 0.5874675876369662, + "f1_weighted": 0.6258355550566823 + }, + { + "accuracy": 0.6857142857142857, + "f1": 0.6544493651670709, + "f1_weighted": 0.6907755482506079 + }, + { + "accuracy": 0.6457142857142857, + "f1": 0.6082372395298248, + "f1_weighted": 0.6587541063110648 + }, + { + "accuracy": 0.6171428571428571, + "f1": 0.5794418353759277, + "f1_weighted": 0.6232329150433427 + } + ] + }, + { + "accuracy": 0.6724285714285714, + "f1": 0.6511835482101187, + "f1_weighted": 0.6755249039430138, + "hf_subset": "mak", + "languages": [ + "mak-Latn" + ], + "main_score": 0.6511835482101187, + "scores_per_experiment": [ + { + "accuracy": 0.69, + "f1": 0.6717625788507542, + "f1_weighted": 0.6948107797216528 + }, + { + "accuracy": 0.6228571428571429, + "f1": 0.5984127933457115, + "f1_weighted": 0.6153350611191378 + }, + { + "accuracy": 0.6128571428571429, + "f1": 0.5952993377079482, + "f1_weighted": 0.6278027362558014 + }, + { + "accuracy": 0.6828571428571428, + "f1": 0.6590623678306446, + "f1_weighted": 0.6920721246352323 + }, + { + "accuracy": 0.7028571428571428, + "f1": 0.6765364771824226, + "f1_weighted": 0.7109664301625358 + }, + { + "accuracy": 0.6542857142857142, + "f1": 0.6381157901178615, + "f1_weighted": 0.6543146080123529 + }, + { + "accuracy": 0.69, + "f1": 0.6644927910614913, + "f1_weighted": 0.6941090188850209 + }, + { + "accuracy": 0.6985714285714286, + "f1": 0.671264262718122, + "f1_weighted": 0.6967453299707091 + }, + { + "accuracy": 0.66, + "f1": 0.6447745208364051, + "f1_weighted": 0.6640419778663437 + }, + { + "accuracy": 0.71, + "f1": 0.6921145624498259, + "f1_weighted": 0.7050509728013513 + } + ] + }, + { + "accuracy": 0.6964999999999999, + "f1": 0.6830295857444562, + "f1_weighted": 0.6915282347216338, + "hf_subset": "min", + "languages": [ + "min-Latn" + ], + "main_score": 0.6830295857444562, + "scores_per_experiment": [ + { + "accuracy": 0.68375, + "f1": 0.6702457537594522, + "f1_weighted": 0.6770744409139724 + }, + { + "accuracy": 0.71375, + "f1": 0.7014287021765294, + "f1_weighted": 0.7114548367533642 + }, + { + "accuracy": 0.68875, + "f1": 0.683918919282406, + "f1_weighted": 0.6859656175643266 + }, + { + "accuracy": 0.67875, + "f1": 0.6568105197110642, + "f1_weighted": 0.6691392787635434 + }, + { + "accuracy": 0.68375, + "f1": 0.6739963016587547, + "f1_weighted": 0.6845920477631463 + }, + { + "accuracy": 0.6825, + "f1": 0.6690417735157195, + "f1_weighted": 0.6757194311478294 + }, + { + "accuracy": 0.7225, + "f1": 0.7115950196187414, + "f1_weighted": 0.7184767185842589 + }, + { + "accuracy": 0.72125, + "f1": 0.7038165870974379, + "f1_weighted": 0.709298748442053 + }, + { + "accuracy": 0.68375, + "f1": 0.6652037875745539, + "f1_weighted": 0.6800049354748845 + }, + { + "accuracy": 0.70625, + "f1": 0.6942384930499023, + "f1_weighted": 0.7035562918089598 + } + ] + }, + { + "accuracy": 0.78625, + "f1": 0.7847577681704051, + "f1_weighted": 0.7827707451596769, + "hf_subset": "mui", + "languages": [ + "mui-Latn" + ], + "main_score": 0.7847577681704051, + "scores_per_experiment": [ + { + "accuracy": 0.8175, + "f1": 0.8136792768116344, + "f1_weighted": 0.8125701012207158 + }, + { + "accuracy": 0.77, + "f1": 0.7639447042089006, + "f1_weighted": 0.769491944554763 + }, + { + "accuracy": 0.765, + "f1": 0.760912949290902, + "f1_weighted": 0.7600984243783456 + }, + { + "accuracy": 0.7825, + "f1": 0.7813158754323395, + "f1_weighted": 0.7804007528036853 + }, + { + "accuracy": 0.7775, + "f1": 0.7783410732714138, + "f1_weighted": 0.7734143262568185 + }, + { + "accuracy": 0.8225, + "f1": 0.8237359479743818, + "f1_weighted": 0.8220814973595704 + }, + { + "accuracy": 0.815, + "f1": 0.8128262921702479, + "f1_weighted": 0.8106348212952583 + }, + { + "accuracy": 0.7775, + "f1": 0.7762081734572419, + "f1_weighted": 0.7741545910545169 + }, + { + "accuracy": 0.7775, + "f1": 0.7806474552515829, + "f1_weighted": 0.7733772607540079 + }, + { + "accuracy": 0.7575, + "f1": 0.7559659338354059, + "f1_weighted": 0.7514837319190876 + } + ] + }, + { + "accuracy": 0.7497142857142858, + "f1": 0.6618562226776146, + "f1_weighted": 0.7367486723643226, + "hf_subset": "rej", + "languages": [ + "rej-Latn" + ], + "main_score": 0.6618562226776146, + "scores_per_experiment": [ + { + "accuracy": 0.7371428571428571, + "f1": 0.6510866939036262, + "f1_weighted": 0.7239660803516531 + }, + { + "accuracy": 0.7628571428571429, + "f1": 0.6736682146254387, + "f1_weighted": 0.7515286713548246 + }, + { + "accuracy": 0.7542857142857143, + "f1": 0.6611847664353725, + "f1_weighted": 0.7389809391291607 + }, + { + "accuracy": 0.7514285714285714, + "f1": 0.6619694132559071, + "f1_weighted": 0.7411733361603973 + }, + { + "accuracy": 0.76, + "f1": 0.6756188990984007, + "f1_weighted": 0.7483000209864257 + }, + { + "accuracy": 0.76, + "f1": 0.6669734883264213, + "f1_weighted": 0.7446652254400593 + }, + { + "accuracy": 0.7342857142857143, + "f1": 0.6479386702247432, + "f1_weighted": 0.720606529215327 + }, + { + "accuracy": 0.7485714285714286, + "f1": 0.6623596989128226, + "f1_weighted": 0.7374543104494233 + }, + { + "accuracy": 0.7228571428571429, + "f1": 0.6421795712432292, + "f1_weighted": 0.7039382185066186 + }, + { + "accuracy": 0.7657142857142857, + "f1": 0.6755828107501833, + "f1_weighted": 0.7568733920493368 + } + ] + }, + { + "accuracy": 0.7243333333333334, + "f1": 0.7114371845316356, + "f1_weighted": 0.7194217975104253, + "hf_subset": "sun", + "languages": [ + "sun-Latn" + ], + "main_score": 0.7114371845316356, + "scores_per_experiment": [ + { + "accuracy": 0.7211111111111111, + "f1": 0.7032167463220967, + "f1_weighted": 0.7150200350748216 + }, + { + "accuracy": 0.73, + "f1": 0.720996646120255, + "f1_weighted": 0.7298742341967669 + }, + { + "accuracy": 0.7011111111111111, + "f1": 0.681590025980304, + "f1_weighted": 0.6949043469007721 + }, + { + "accuracy": 0.7011111111111111, + "f1": 0.6862001242253536, + "f1_weighted": 0.6967260831325527 + }, + { + "accuracy": 0.7744444444444445, + "f1": 0.7686245429585276, + "f1_weighted": 0.7696565484178968 + }, + { + "accuracy": 0.7377777777777778, + "f1": 0.7298825211940584, + "f1_weighted": 0.7401893793081832 + }, + { + "accuracy": 0.6766666666666666, + "f1": 0.6649937099597808, + "f1_weighted": 0.6719085635743007 + }, + { + "accuracy": 0.7444444444444445, + "f1": 0.7289737857535564, + "f1_weighted": 0.734925881023402 + }, + { + "accuracy": 0.7633333333333333, + "f1": 0.7527381552976653, + "f1_weighted": 0.7609444235963128 + }, + { + "accuracy": 0.6933333333333334, + "f1": 0.6771555875047582, + "f1_weighted": 0.6800684798792455 + } + ] + } + ] + }, + "task_name": "NusaParagraphTopicClassification" +} \ No newline at end of file diff --git a/results/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/bf3bf13ab40c3157080a7ab344c831b9ad18b5eb/NusaParagraphTopicClassification.json b/results/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/bf3bf13ab40c3157080a7ab344c831b9ad18b5eb/NusaParagraphTopicClassification.json new file mode 100644 index 0000000000..3628506b52 --- /dev/null +++ b/results/sentence-transformers__paraphrase-multilingual-MiniLM-L12-v2/bf3bf13ab40c3157080a7ab344c831b9ad18b5eb/NusaParagraphTopicClassification.json @@ -0,0 +1,631 @@ +{ + "dataset_revision": "abb43f8d5b9510b8724b48283aca26c4733eac5d", + "evaluation_time": 21.640820264816284, + "kg_co2_emissions": null, + "mteb_version": "1.12.30", + "scores": { + "test": [ + { + "accuracy": 0.502, + "f1": 0.49583124915326754, + "f1_weighted": 0.5049909818870219, + "hf_subset": "btk", + "languages": [ + "bbc-Latn" + ], + "main_score": 0.49583124915326754, + "scores_per_experiment": [ + { + "accuracy": 0.494, + "f1": 0.4923836681373438, + "f1_weighted": 0.502604128657887 + }, + { + "accuracy": 0.486, + "f1": 0.4957754504845821, + "f1_weighted": 0.49212831242694527 + }, + { + "accuracy": 0.498, + "f1": 0.48527722327799533, + "f1_weighted": 0.4987207967306596 + }, + { + "accuracy": 0.532, + "f1": 0.5146508810932297, + "f1_weighted": 0.5245154025413195 + }, + { + "accuracy": 0.516, + "f1": 0.5022581319237363, + "f1_weighted": 0.5076281198283419 + }, + { + "accuracy": 0.522, + "f1": 0.5161932075375283, + "f1_weighted": 0.5252784631941385 + }, + { + "accuracy": 0.494, + "f1": 0.4925507028978733, + "f1_weighted": 0.5096481832965343 + }, + { + "accuracy": 0.52, + "f1": 0.5058674282430002, + "f1_weighted": 0.5219240914691716 + }, + { + "accuracy": 0.474, + "f1": 0.46950643124956726, + "f1_weighted": 0.4886392403843932 + }, + { + "accuracy": 0.484, + "f1": 0.4838493666878183, + "f1_weighted": 0.4788230803408273 + } + ] + }, + { + "accuracy": 0.623875, + "f1": 0.609485091619269, + "f1_weighted": 0.6323509147743607, + "hf_subset": "bew", + "languages": [ + "bew-Latn" + ], + "main_score": 0.609485091619269, + "scores_per_experiment": [ + { + "accuracy": 0.5975, + "f1": 0.5916211091826291, + "f1_weighted": 0.617688274045643 + }, + { + "accuracy": 0.62875, + "f1": 0.6166869918521367, + "f1_weighted": 0.6466030104992363 + }, + { + "accuracy": 0.6675, + "f1": 0.6508769667597738, + "f1_weighted": 0.6668098124434177 + }, + { + "accuracy": 0.66625, + "f1": 0.6451817470042513, + "f1_weighted": 0.6709164840710699 + }, + { + "accuracy": 0.58875, + "f1": 0.5779693271763158, + "f1_weighted": 0.5950794139326674 + }, + { + "accuracy": 0.635, + "f1": 0.6212669930801686, + "f1_weighted": 0.6482536445815563 + }, + { + "accuracy": 0.6225, + "f1": 0.6056062227188277, + "f1_weighted": 0.6362560522316104 + }, + { + "accuracy": 0.6025, + "f1": 0.582070219968576, + "f1_weighted": 0.60118405063164 + }, + { + "accuracy": 0.5675, + "f1": 0.5536907573182387, + "f1_weighted": 0.5691135337806049 + }, + { + "accuracy": 0.6625, + "f1": 0.6498805811317716, + "f1_weighted": 0.6716048715261607 + } + ] + }, + { + "accuracy": 0.5686666666666668, + "f1": 0.5184796739036743, + "f1_weighted": 0.5576343730542004, + "hf_subset": "bug", + "languages": [ + "bug-Latn" + ], + "main_score": 0.5184796739036743, + "scores_per_experiment": [ + { + "accuracy": 0.5666666666666667, + "f1": 0.5091118896430855, + "f1_weighted": 0.5529787709918204 + }, + { + "accuracy": 0.5533333333333333, + "f1": 0.5102326371014173, + "f1_weighted": 0.5495934250015552 + }, + { + "accuracy": 0.55, + "f1": 0.5017254073738868, + "f1_weighted": 0.5435945549599398 + }, + { + "accuracy": 0.5966666666666667, + "f1": 0.5334320550375442, + "f1_weighted": 0.5786127152016327 + }, + { + "accuracy": 0.5566666666666666, + "f1": 0.5204244932192152, + "f1_weighted": 0.5452414686554135 + }, + { + "accuracy": 0.5633333333333334, + "f1": 0.5216067410039114, + "f1_weighted": 0.5540161648606108 + }, + { + "accuracy": 0.5933333333333334, + "f1": 0.5326338751019488, + "f1_weighted": 0.5819798352602745 + }, + { + "accuracy": 0.58, + "f1": 0.5250692251271024, + "f1_weighted": 0.5642819856464932 + }, + { + "accuracy": 0.5566666666666666, + "f1": 0.5148036197321377, + "f1_weighted": 0.5493701772027021 + }, + { + "accuracy": 0.57, + "f1": 0.515756795696493, + "f1_weighted": 0.5566746327615613 + } + ] + }, + { + "accuracy": 0.498625, + "f1": 0.484702668509168, + "f1_weighted": 0.5080896250345485, + "hf_subset": "jav", + "languages": [ + "jav-Latn" + ], + "main_score": 0.484702668509168, + "scores_per_experiment": [ + { + "accuracy": 0.47625, + "f1": 0.46738222141513686, + "f1_weighted": 0.48301727833616664 + }, + { + "accuracy": 0.485, + "f1": 0.4753544376312615, + "f1_weighted": 0.4953950215027018 + }, + { + "accuracy": 0.49625, + "f1": 0.48459195024714796, + "f1_weighted": 0.5015912846827465 + }, + { + "accuracy": 0.52625, + "f1": 0.5115138176602833, + "f1_weighted": 0.5383827968174302 + }, + { + "accuracy": 0.495, + "f1": 0.4804018819929509, + "f1_weighted": 0.5053251777500593 + }, + { + "accuracy": 0.51625, + "f1": 0.49999253357728735, + "f1_weighted": 0.517854193724271 + }, + { + "accuracy": 0.4625, + "f1": 0.45619860700438875, + "f1_weighted": 0.4871324617537307 + }, + { + "accuracy": 0.49375, + "f1": 0.4718442332968104, + "f1_weighted": 0.5033444838913722 + }, + { + "accuracy": 0.5175, + "f1": 0.5013059251492795, + "f1_weighted": 0.5248308426522561 + }, + { + "accuracy": 0.5175, + "f1": 0.49844107711713403, + "f1_weighted": 0.5240227092347505 + } + ] + }, + { + "accuracy": 0.4578571428571429, + "f1": 0.4293434328627389, + "f1_weighted": 0.474337380884348, + "hf_subset": "mad", + "languages": [ + "mad-Latn" + ], + "main_score": 0.4293434328627389, + "scores_per_experiment": [ + { + "accuracy": 0.4142857142857143, + "f1": 0.36830308087843344, + "f1_weighted": 0.4231514842820036 + }, + { + "accuracy": 0.4757142857142857, + "f1": 0.4311602002297987, + "f1_weighted": 0.4869944417098914 + }, + { + "accuracy": 0.43142857142857144, + "f1": 0.3963653473783615, + "f1_weighted": 0.4346653723331879 + }, + { + "accuracy": 0.4442857142857143, + "f1": 0.4315235896654613, + "f1_weighted": 0.4793797268903201 + }, + { + "accuracy": 0.44571428571428573, + "f1": 0.4313647527747436, + "f1_weighted": 0.4676159906990612 + }, + { + "accuracy": 0.44, + "f1": 0.4159285938894658, + "f1_weighted": 0.44753026080079944 + }, + { + "accuracy": 0.4785714285714286, + "f1": 0.46335781606938115, + "f1_weighted": 0.5037925549493945 + }, + { + "accuracy": 0.5228571428571429, + "f1": 0.49904675560668343, + "f1_weighted": 0.5389392474919658 + }, + { + "accuracy": 0.47285714285714286, + "f1": 0.4427125121202358, + "f1_weighted": 0.48814505670401 + }, + { + "accuracy": 0.45285714285714285, + "f1": 0.41367168001482396, + "f1_weighted": 0.4731596729828466 + } + ] + }, + { + "accuracy": 0.5084285714285715, + "f1": 0.48917602380236913, + "f1_weighted": 0.5202075190662195, + "hf_subset": "mak", + "languages": [ + "mak-Latn" + ], + "main_score": 0.48917602380236913, + "scores_per_experiment": [ + { + "accuracy": 0.5128571428571429, + "f1": 0.4902509671033717, + "f1_weighted": 0.5251142347684558 + }, + { + "accuracy": 0.5128571428571429, + "f1": 0.4911172384032806, + "f1_weighted": 0.5173883280980776 + }, + { + "accuracy": 0.45285714285714285, + "f1": 0.4405715557485829, + "f1_weighted": 0.4701317315942821 + }, + { + "accuracy": 0.49714285714285716, + "f1": 0.4797791249551845, + "f1_weighted": 0.5097664107503609 + }, + { + "accuracy": 0.5514285714285714, + "f1": 0.5186223018361971, + "f1_weighted": 0.55930276399965 + }, + { + "accuracy": 0.47714285714285715, + "f1": 0.46146332809137885, + "f1_weighted": 0.49200874830422736 + }, + { + "accuracy": 0.5157142857142857, + "f1": 0.5059612429055371, + "f1_weighted": 0.5335933140210743 + }, + { + "accuracy": 0.5385714285714286, + "f1": 0.5068910053350097, + "f1_weighted": 0.5453236601032398 + }, + { + "accuracy": 0.49142857142857144, + "f1": 0.48385599556216274, + "f1_weighted": 0.4992476422558179 + }, + { + "accuracy": 0.5342857142857143, + "f1": 0.5132474780829864, + "f1_weighted": 0.5501983567670095 + } + ] + }, + { + "accuracy": 0.599125, + "f1": 0.592963903953233, + "f1_weighted": 0.6033851196187611, + "hf_subset": "min", + "languages": [ + "min-Latn" + ], + "main_score": 0.592963903953233, + "scores_per_experiment": [ + { + "accuracy": 0.59125, + "f1": 0.5827173261341706, + "f1_weighted": 0.596372571213758 + }, + { + "accuracy": 0.63, + "f1": 0.6165168681838411, + "f1_weighted": 0.6321415731411145 + }, + { + "accuracy": 0.59125, + "f1": 0.5882802011720217, + "f1_weighted": 0.5975118239207026 + }, + { + "accuracy": 0.59375, + "f1": 0.5780553154747631, + "f1_weighted": 0.5957930597057401 + }, + { + "accuracy": 0.565, + "f1": 0.5777521634408125, + "f1_weighted": 0.577006119572099 + }, + { + "accuracy": 0.56875, + "f1": 0.5735399420881627, + "f1_weighted": 0.5712645373956117 + }, + { + "accuracy": 0.64625, + "f1": 0.6247343172455967, + "f1_weighted": 0.6462731212642467 + }, + { + "accuracy": 0.64, + "f1": 0.6333319506660035, + "f1_weighted": 0.6425678101934124 + }, + { + "accuracy": 0.58625, + "f1": 0.5789297989013357, + "f1_weighted": 0.5919365192699684 + }, + { + "accuracy": 0.57875, + "f1": 0.5757811562256222, + "f1_weighted": 0.5829840605109579 + } + ] + }, + { + "accuracy": 0.70775, + "f1": 0.7095409755484919, + "f1_weighted": 0.7087379049270817, + "hf_subset": "mui", + "languages": [ + "mui-Latn" + ], + "main_score": 0.7095409755484919, + "scores_per_experiment": [ + { + "accuracy": 0.7225, + "f1": 0.7181470763951315, + "f1_weighted": 0.7193019953884903 + }, + { + "accuracy": 0.705, + "f1": 0.7051496104917943, + "f1_weighted": 0.7096501532593039 + }, + { + "accuracy": 0.715, + "f1": 0.716795819197157, + "f1_weighted": 0.716824773006319 + }, + { + "accuracy": 0.6975, + "f1": 0.6994367470218236, + "f1_weighted": 0.6974375682615824 + }, + { + "accuracy": 0.7225, + "f1": 0.7218389671504905, + "f1_weighted": 0.722994535231039 + }, + { + "accuracy": 0.7, + "f1": 0.7063910617310575, + "f1_weighted": 0.7032539841645798 + }, + { + "accuracy": 0.7075, + "f1": 0.7159542679163657, + "f1_weighted": 0.7087279623828745 + }, + { + "accuracy": 0.6825, + "f1": 0.6888119175399933, + "f1_weighted": 0.6828403776671707 + }, + { + "accuracy": 0.705, + "f1": 0.7041029633721217, + "f1_weighted": 0.7057358718482053 + }, + { + "accuracy": 0.72, + "f1": 0.7187813246689843, + "f1_weighted": 0.7206118280612531 + } + ] + }, + { + "accuracy": 0.65, + "f1": 0.6511645155269374, + "f1_weighted": 0.652331326218501, + "hf_subset": "rej", + "languages": [ + "rej-Latn" + ], + "main_score": 0.6511645155269374, + "scores_per_experiment": [ + { + "accuracy": 0.6771428571428572, + "f1": 0.675208982723501, + "f1_weighted": 0.6777449602546418 + }, + { + "accuracy": 0.6542857142857142, + "f1": 0.6500828083159753, + "f1_weighted": 0.6569148737908412 + }, + { + "accuracy": 0.6314285714285715, + "f1": 0.6348749406891112, + "f1_weighted": 0.6293023000727321 + }, + { + "accuracy": 0.6828571428571428, + "f1": 0.6807494517074276, + "f1_weighted": 0.6858182469697525 + }, + { + "accuracy": 0.64, + "f1": 0.6497710882318689, + "f1_weighted": 0.6463249580751427 + }, + { + "accuracy": 0.64, + "f1": 0.6416243820254892, + "f1_weighted": 0.6429522457917923 + }, + { + "accuracy": 0.64, + "f1": 0.6391267680157248, + "f1_weighted": 0.6442285014250654 + }, + { + "accuracy": 0.6685714285714286, + "f1": 0.6772072785668375, + "f1_weighted": 0.6744002548060357 + }, + { + "accuracy": 0.64, + "f1": 0.6280601382517603, + "f1_weighted": 0.6372182741711646 + }, + { + "accuracy": 0.6257142857142857, + "f1": 0.6349393167416787, + "f1_weighted": 0.6284086468278409 + } + ] + }, + { + "accuracy": 0.5413333333333333, + "f1": 0.5376816868659201, + "f1_weighted": 0.5510306569469691, + "hf_subset": "sun", + "languages": [ + "sun-Latn" + ], + "main_score": 0.5376816868659201, + "scores_per_experiment": [ + { + "accuracy": 0.5155555555555555, + "f1": 0.5124935121451668, + "f1_weighted": 0.5257431334680848 + }, + { + "accuracy": 0.5522222222222222, + "f1": 0.5505331734375645, + "f1_weighted": 0.5641846244150112 + }, + { + "accuracy": 0.5522222222222222, + "f1": 0.5471253203820344, + "f1_weighted": 0.567516360357803 + }, + { + "accuracy": 0.54, + "f1": 0.5288053651337596, + "f1_weighted": 0.5406112101952358 + }, + { + "accuracy": 0.5288888888888889, + "f1": 0.5330090553749861, + "f1_weighted": 0.5408310949835916 + }, + { + "accuracy": 0.5577777777777778, + "f1": 0.5595453801208847, + "f1_weighted": 0.5712989239500295 + }, + { + "accuracy": 0.5033333333333333, + "f1": 0.5009258996416388, + "f1_weighted": 0.5113015355658886 + }, + { + "accuracy": 0.5622222222222222, + "f1": 0.5608093274440646, + "f1_weighted": 0.5735566083414523 + }, + { + "accuracy": 0.5444444444444444, + "f1": 0.5383340120878801, + "f1_weighted": 0.5539581073844755 + }, + { + "accuracy": 0.5566666666666666, + "f1": 0.5452358228912215, + "f1_weighted": 0.561304970808119 + } + ] + } + ] + }, + "task_name": "NusaParagraphTopicClassification" +} \ No newline at end of file