Only pass the use_kv_cache True to generator (huggingface#1366)

Co-authored-by: regisss <15324346+regisss@users.noreply.github.com>
Co-authored-by: Vidya Galli <vidya.s.galli@intel.com>
Co-authored-by: kaixuanliu <kaixuan.liu@intel.com>

Author: 4 people

examples/image-to-text/run_pipeline.py

@@ -211,13 +211,15 @@ def main():
     )
     generate_kwargs = {
         "lazy_mode": True,
-        "use_cache": args.use_kv_cache,
         "hpu_graphs": args.use_hpu_graphs,
         "max_new_tokens": args.max_new_tokens,
         "ignore_eos": args.ignore_eos,
         "use_flash_attention": args.use_flash_attention,
         "flash_attention_recompute": args.flash_attention_recompute,
     }
+    if args.use_kv_cache:
+        generate_kwargs["use_cache"] = args.use_kv_cache
+
     if args.use_hpu_graphs:
         from habana_frameworks.torch.hpu import wrap_in_hpu_graph
 

tests/test_image_to_text_example.py

@@ -14,15 +14,15 @@
     # Gaudi2 CI baselines
     MODELS_TO_TEST = {
         "bf16": [
-            ("llava-hf/llava-1.5-7b-hf", 1, 87.2901500056982),
-            ("llava-hf/llava-1.5-13b-hf", 1, 51.04717105443364),
+            ("llava-hf/llava-1.5-7b-hf", 1, 77.98733740859008),
+            ("llava-hf/llava-1.5-13b-hf", 1, 48.54364937033955),
             ("llava-hf/llava-v1.6-mistral-7b-hf", 1, 33.17984878151546),
             ("llava-hf/llava-v1.6-vicuna-7b-hf", 1, 35.00608681379742),
             ("llava-hf/llava-v1.6-vicuna-13b-hf", 1, 23.527610042925),
         ],
         "fp8": [
-            ("llava-hf/llava-1.5-7b-hf", 1, 115.48515989461843),
-            ("llava-hf/llava-1.5-13b-hf", 1, 78.2635142547838),
+            ("llava-hf/llava-1.5-7b-hf", 1, 98.72578382705062),
+            ("llava-hf/llava-1.5-13b-hf", 1, 67.20488222876344),
             ("llava-hf/llava-v1.6-mistral-7b-hf", 1, 45.011551008367084),
             ("llava-hf/llava-v1.6-vicuna-7b-hf", 1, 45.18544502949674),
             ("llava-hf/llava-v1.6-vicuna-13b-hf", 1, 30.9535718774675),