diff --git a/mteb/models/gme_models.py b/mteb/models/gme_models.py index 53476403b..804dfbc84 100644 --- a/mteb/models/gme_models.py +++ b/mteb/models/gme_models.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -from functools import partial from mteb.model_meta import ModelMeta diff --git a/mteb/models/lens_models.py b/mteb/models/lens_models.py index 2fe54b26a..380724e53 100644 --- a/mteb/models/lens_models.py +++ b/mteb/models/lens_models.py @@ -1,12 +1,6 @@ from __future__ import annotations -from functools import partial - -import torch - -from mteb.encoder_interface import PromptType -from mteb.model_meta import ModelMeta, sentence_transformers_loader -from mteb.models.instruct_wrapper import instruct_wrapper +from mteb.model_meta import ModelMeta from .bge_models import bge_full_data diff --git a/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py b/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py index d60b7a381..92a489c01 100644 --- a/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py +++ b/mteb/tasks/Retrieval/eng/ClimateFEVERRetrieval.py @@ -72,3 +72,39 @@ class ClimateFEVERHardNegatives(AbsTaskRetrieval): primaryClass={cs.CL} }""", ) + + +class ClimateFEVERv2(AbsTaskRetrieval): + metadata = TaskMetadata( + name="ClimateFEVER.v2", + description="CLIMATE-FEVER is a dataset adopting the FEVER methodology that consists of 1,535 real-world claims regarding climate-change. ", + reference="https://www.sustainablefinance.uzh.ch/en/research/climate-fever.html", + dataset={ + "path": "Mina76/climate-fever", + "revision": "e438c9586767800aeb10dbe8a245c41dbea4e5f4", + }, + type="Retrieval", + category="s2p", + modalities=["text"], + eval_splits=["test"], + eval_langs=["eng-Latn"], + main_score="ndcg_at_10", + date=("2020-12-11", "2020-12-11"), + domains=["Academic"], + task_subtypes=["Question answering"], + license="cc-by-sa-4.0", + annotations_creators="human-annotated", + dialect=[], + sample_creation="found", + bibtex_citation="""@misc{diggelmann2021climatefever, + title={CLIMATE-FEVER: A Dataset for Verification of Real-World Climate Claims}, + author={Thomas Diggelmann and Jordan Boyd-Graber and Jannis Bulian and Massimiliano Ciaramita and Markus Leippold}, + year={2021}, + eprint={2012.00614}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +}""", + prompt={ + "query": "Given a claim about climate change, retrieve documents that support or refute the claim" + }, + )