Fix some small bugs (EvolvingLMMs-Lab#101)

* Refactor GeminiAPI to use UUIDs for caching * remove useless code
kangreen0210 · May 28, 2024 · 8fdd47e · 8fdd47e
1 parent 1e1c17c
commit 8fdd47e
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 11 deletions.
diff --git a/lmms_eval/models/gemini_api.py b/lmms_eval/models/gemini_api.py
@@ -110,10 +110,13 @@ def convert_video(self, images):
     def generate_until(self, requests) -> List[str]:
         res = []
         pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
+
+        def get_uuid(task, split, doc_id):
+            return f"{task}___{split}___{doc_id}"
 
         for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
             if self.continual_mode is True and self.cache_mode == "resume":
-                doc_uuid = str(doc_id)
+                doc_uuid = get_uuid(task, split, doc_id)
                 if doc_uuid in self.response_cache:
                     content = self.response_cache[doc_uuid]
                     if content:
@@ -169,7 +172,7 @@ def generate_until(self, requests) -> List[str]:
             pbar.update(1)
 
             if self.continual_mode is True:  # Cache the response
-                doc_uuid = str(doc_id)
+                doc_uuid = get_uuid(task, split, doc_id)
                 self.response_cache[doc_uuid] = content
                 with open(self.response_persistent_file, "w") as f:
                     json.dump(self.response_cache, f)

diff --git a/lmms_eval/tasks/nextqa/utils.py b/lmms_eval/tasks/nextqa/utils.py
@@ -198,13 +198,13 @@ def nextqa_aggregate_results(results):
             wups0[qtype] = 0
             wups9[qtype] = 0
 
-    num_e = over_num["C"]
-    num_t = over_num["T"]
-    num_c = over_num["D"]
+    # num_e = over_num["C"]
+    # num_t = over_num["T"]
+    # num_c = over_num["D"]
 
-    wups0_e /= num_e
-    wups0_t /= num_t
-    wups0_c /= num_c
+    # wups0_e /= num_e
+    # wups0_t /= num_t
+    # wups0_c /= num_c
 
     wups0_all /= ref_num
     wups9_all /= ref_num
@@ -213,9 +213,9 @@ def nextqa_aggregate_results(results):
         wups0[k] = wups0[k] * 100
         wups9[k] = wups9[k] * 100
 
-    wups0_e *= 100
-    wups0_t *= 100
-    wups0_c *= 100
+    # wups0_e *= 100
+    # wups0_t *= 100
+    # wups0_c *= 100
     wups0_all *= 100
 
     return wups0_all