diff --git a/lmms_eval/tasks/pope/pope.yaml b/lmms_eval/tasks/pope/pope.yaml
index 2450201cb..703fe3d82 100644
--- a/lmms_eval/tasks/pope/pope.yaml
+++ b/lmms_eval/tasks/pope/pope.yaml
@@ -8,8 +8,11 @@ doc_to_visual: !function utils.pope_doc_to_visual
 doc_to_text: !function utils.pope_doc_to_text
 doc_to_target: "answer"
 generation_kwargs:
-  until:
-    - "ASSISTANT:"
+  max_new_tokens: 128
+  temperature: 0
+  top_p: 0
+  num_beams: 1
+  do_sample: false
 process_results: !function utils.pope_process_results
 metric_list:
   - metric: pope_accuracy
diff --git a/lmms_eval/tasks/pope/utils.py b/lmms_eval/tasks/pope/utils.py
index f33485c47..34f749248 100644
--- a/lmms_eval/tasks/pope/utils.py
+++ b/lmms_eval/tasks/pope/utils.py
@@ -9,7 +9,7 @@ def pope_doc_to_visual(doc):
 def pope_doc_to_text(doc):
     # Assuming the 'doc' dictionary has a key 'question' with the question text
     question = doc["question"].strip()
-    return f"{question}\nAnswer the question with 'yes' or 'no'."
+    return f"{question}\nAnswer the question using a single word or phrase."
 
 
 def pope_process_results(doc, results):