[Datasets] Added POPE and Aligned. (#11)

* Update generation_kwargs in pope.yaml * Update pope_doc_to_text function
EvolvingLMMs-Lab · Jan 23, 2024 · 4c3c2c6 · 4c3c2c6
1 parent b025c0e
commit 4c3c2c6
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 3 deletions.
diff --git a/lmms_eval/tasks/pope/pope.yaml b/lmms_eval/tasks/pope/pope.yaml
@@ -8,8 +8,11 @@ doc_to_visual: !function utils.pope_doc_to_visual
 doc_to_text: !function utils.pope_doc_to_text
 doc_to_target: "answer"
 generation_kwargs:
-  until:
-    - "ASSISTANT:"
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
 process_results: !function utils.pope_process_results
 metric_list:
   - metric: pope_accuracy

diff --git a/lmms_eval/tasks/pope/utils.py b/lmms_eval/tasks/pope/utils.py
@@ -9,7 +9,7 @@ def pope_doc_to_visual(doc):
 def pope_doc_to_text(doc):
     # Assuming the 'doc' dictionary has a key 'question' with the question text
     question = doc["question"].strip()
-    return f"{question}\nAnswer the question with 'yes' or 'no'."
+    return f"{question}\nAnswer the question using a single word or phrase."
 
 
 def pope_process_results(doc, results):