fix: Add error reporting for Retrieval

mteb/abstasks/AbsTaskRetrieval.py

@@ -285,10 +285,14 @@ def _evaluate_subset(
             "Time taken to retrieve: {:.2f} seconds".format(end_time - start_time)
         )
 
-        if kwargs.get("save_predictions", False):
+        save_predictions = kwargs.get("save_predictions", False)
+        export_errors = kwargs.get("export_errors", False)
+        if save_predictions or export_errors:
             output_folder = Path(kwargs.get("output_folder", "results"))
             if not os.path.isdir(output_folder):
                 os.makedirs(output_folder)
+
+        if save_predictions:
             top_k = kwargs.get("top_k", None)
             if top_k is not None:
                 for qid in list(results.keys()):
@@ -301,8 +305,7 @@ def _evaluate_subset(
                         k: v for k, v in results[qid].items() if k in doc_ids
                     }
             qrels_save_path = (
-                output_folder
-                / f"{self.metadata_dict['name']}_{hf_subset}_predictions.json"
+                output_folder / f"{self.metadata.name}_{hf_subset}_predictions.json"
             )
 
             with open(qrels_save_path, "w") as f:
@@ -333,6 +336,38 @@ def _evaluate_subset(
             },
         }
         self._add_main_score(scores)
+
+        if export_errors:
+            errors = {}
+
+            top_k = kwargs.get("top_k", 1)
+            if not save_predictions and top_k == 1:
+                for qid in results.keys():
+                    doc_scores = results[qid]
+                    sorted_docs = sorted(
+                        doc_scores.items(), key=lambda x: x[1], reverse=True
+                    )[:top_k]
+                    results[qid] = {doc_id: score for doc_id, score in sorted_docs}
+            for qid, retrieved_docs in results.items():
+                expected_docs = relevant_docs[qid]
+                false_positives = [
+                    doc for doc in retrieved_docs if doc not in expected_docs
+                ]
+                false_negatives = [
+                    doc for doc in expected_docs if doc not in retrieved_docs
+                ]
+                if false_positives or false_negatives:
+                    errors[qid] = {
+                        "false_positives": false_positives,
+                        "false_negatives": false_negatives,
+                    }
+
+            errors_save_path = (
+                output_folder / f"{self.metadata.name}_{hf_subset}_errors.json"
+            )
+            with open(errors_save_path, "w") as f:
+                json.dump(errors, f)
+
         return scores
 
     def _add_main_score(self, scores: ScoresDict) -> None: