Merge pull request #145 from Multi-Agent-LLMs/aqua_rat

Aqua rat

README.md

@@ -32,7 +32,7 @@ Install as a package:
 ### Create Data
 Download and create the test data: `python data/data_downloader.py --datasets=[SQuAD2,ETPC]`
 
-You can use any dataset for this project as long as it follows [this basic format](https://github.com/Multi-Agent-LLMs/mallm/blob/main/data/datasets/etpc_debugging.json). These datasets are supported by our automated formatting pipeline: `BBQGenderIdentity`, `BTVote`, `ETHICS`, `ETPC`, `Europarl`, `GPQA`, `GSM8K`, `IFEval`, `MMLU`, `MMLUPro`, `MUSR`, `MathLvl5`, `MoCaMoral`, `MoralExceptQA`, `MultiNews`, `SQuAD2`, `SimpleEthicalQuestions`, `StrategyQA`, `WMT19DeEn`, `WinoGrande`, `XSum`
+You can use any dataset for this project as long as it follows [this basic format](https://github.com/Multi-Agent-LLMs/mallm/blob/main/data/datasets/etpc_debugging.json). These datasets are supported by our automated formatting pipeline: `AquaRat`, `BBQGenderIdentity`, `BTVote`, `ETHICS`, `ETPC`, `Europarl`, `GPQA`, `GSM8K`, `IFEval`, `MMLU`, `MMLUPro`, `MUSR`, `MathLvl5`, `MoCaMoral`, `MoralExceptQA`, `MultiNews`, `SQuAD2`, `SimpleEthicalQuestions`, `StrategyQA`, `WMT19DeEn`, `WinoGrande`, `XSum`
 
 ### Run from Terminal
 MALLM relies on an external API like OpenAI or Text Generation Inference by Huggingface.

data/data_download.py

@@ -122,14 +122,14 @@ def load_and_execute_downloaders(
 ):
     # Path to the directory containing downloader modules
     base_path = Path(__file__).parent / directory
+    executed_downloaders = 0
     # Iterate over each file in the directory
     for file in base_path.glob("*.py"):
         if (
-            file.name == "__init__.py"
-            or datasets
-            and file.name.split(".")[0] not in datasets
+            file.name == "__init__.py" or datasets and file.name.split(".")[0] not in datasets
         ):
             continue
+        executed_downloaders += 1
         print(f"\n\033[96m[PROCESSING]\033[0m Processing {file.name}")
         module_name = f"{directory}.{file.stem}"
         # Dynamically import the module
@@ -152,7 +152,7 @@ def load_and_execute_downloaders(
         else:
             print(f"\033[93m[WARNING]\033[0m No downloader class found in {file.name}")
         print("\033[90m" + "-" * 40 + "\033[0m")  # Separator for clarity
-
-
+    if len(datasets) != executed_downloaders:
+        print(f"\033[93m[WARNING]\033[0m Some of the datasets were not downloaded. Please also check their spelling (including capitalization).\nThese datasets are available: {", ".join([file.name.replace(".py","") for file in base_path.glob("*.py")][:-1])}")
 if __name__ == "__main__":
     fire.Fire(load_and_execute_downloaders)

data/data_downloaders/AquaRat.py

@@ -0,0 +1,46 @@
+import uuid
+from typing import Optional
+
+from data.data_download import DatasetDownloader
+from mallm.utils.types import InputExample
+
+
+class MMLUProDownloader(DatasetDownloader):
+    def custom_download(self):
+        pass
+
+    def __init__(
+        self, sample_size: Optional[int] = None, hf_token: Optional[str] = None, trust_remote_code: bool = False
+    ):
+        super().__init__(
+            name="aqua_rat",
+            dataset_name="deepmind/aqua_rat",
+            version="raw",
+            sample_size=sample_size,
+            hf_token=hf_token,
+            trust_remote_code=trust_remote_code
+        )
+
+    def process_data(self) -> list[InputExample]:
+        data = self.shuffle_and_select("test")
+        input_examples = []
+
+        for sample in data.iter(batch_size=1):
+            answers = sample["options"][0]
+            formatted_answers = [answer.replace(")", ") ", 1) for answer in answers]
+            correct_answer = sample["correct"][0]
+            correct_answer_text = next(answer for answer in formatted_answers if answer.startswith(correct_answer))
+
+            question_text = self._clean_text(sample["question"][0])
+            question_and_answers = f"{question_text}\n\n" + "\n".join(formatted_answers)
+
+            input_examples.append(
+                InputExample(
+                    example_id=str(uuid.uuid4()),
+                    dataset_id=None,
+                    inputs=[question_and_answers],
+                    context=None,
+                    references=[correct_answer_text],
+                )
+            )
+        return input_examples

mallm/utils/task_instructions.py

@@ -11,5 +11,6 @@
     "math_lvl5": "Answer the following math question. Output the number as the final solution.",
     "ifeval": "Follow the instruction.",
     "winogrande": "Correctly fill the blank space _ by choosing option A) or B). Include the letter corresponding to your answer in the solution.",
-    "ethics": "Would most people believe this reasonable or unreasonable to say? Answer with A) Reasonable or B) Unreasonable. Include the letter corresponding to your answer in the solution."
+    "ethics": "Would most people believe this reasonable or unreasonable to say? Answer with A) Reasonable or B) Unreasonable. Include the letter corresponding to your answer in the solution.",
+    "aqua_rat": "Answer the following math question. Include the letter corresponding to your answer in the solution."
 }