openai model updates

- make sure gpt_model and gpt_model_schematic share same set of models - update valid multimodal models - update various costs Signed-off-by: Jack Luar <39641663+luarss@users.noreply.github.com>
confident-ai · Jan 19, 2025 · 5be2d4c · 5be2d4c
1 parent 9effc76
commit 5be2d4c
Show file tree

Hide file tree

Showing 2 changed files with 64 additions and 35 deletions.
diff --git a/deepeval/models/gpt_model.py b/deepeval/models/gpt_model.py
@@ -27,55 +27,70 @@ def log_retry_error(retry_state):
 
 
 valid_gpt_models = [
-    "gpt-4o-mini",
-    "gpt-4o",
-    "gpt-4-turbo",
-    "gpt-4-turbo-preview",
+    "gpt-3.5-turbo",
+    "gpt-3.5-turbo-0125",
+    "gpt-3.5-turbo-1106",
+    "gpt-3.5-turbo-16k",
+    "gpt-3.5-turbo-instruct",
+    "gpt-3.5-turbo-instruct-0914",
+    "gpt-4",
     "gpt-4-0125-preview",
+    "gpt-4-0613",
     "gpt-4-1106-preview",
-    "gpt-4",
+    "gpt-4-turbo",
+    "gpt-4-turbo-2024-04-09",
+    "gpt-4-turbo-preview",
+    "gpt-4o",
+    "gpt-4o-2024-05-13",
+    "gpt-4o-2024-08-06",
+    "gpt-4o-2024-11-20",
+    "gpt-4o-mini",
+    "gpt-4o-mini-2024-07-18",
     "gpt-4-32k",
-    "gpt-4-0613",
     "gpt-4-32k-0613",
-    "gpt-3.5-turbo-1106",
-    "gpt-3.5-turbo",
-    "gpt-3.5-turbo-16k",
-    "gpt-3.5-turbo-0125",
 ]
 
 structured_outputs_models = [
     "gpt-4o",
-    "gpt-4o-mini"
+    "gpt-4o-2024-05-13",
+    "gpt-4o-2024-08-06",
+    "gpt-4o-2024-11-20",
+    "gpt-4o-mini",
+    "gpt-4o-mini-2024-07-18",
 ]
 
 json_mode_models = [
-    "gpt-4-turbo",
-    "gpt-4-turbo-preview",
+    "gpt-3.5-turbo",
+    "gpt-3.5-turbo-0125",
+    "gpt-3.5-turbo-1106",
+    "gpt-3.5-turbo-16k",
+    "gpt-3.5-turbo-instruct",
+    "gpt-3.5-turbo-instruct-0914",
+    "gpt-4",
     "gpt-4-0125-preview",
+    "gpt-4-0613",
     "gpt-4-1106-preview",
-    "gpt-4",
+    "gpt-4-turbo",
+    "gpt-4-turbo-2024-04-09",
+    "gpt-4-turbo-preview",
     "gpt-4-32k",
-    "gpt-4-0613",
     "gpt-4-32k-0613",
-    "gpt-3.5-turbo-1106",
-    "gpt-3.5-turbo",
-    "gpt-3.5-turbo-16k",
-    "gpt-3.5-turbo-0125",
 ]
 
 model_pricing = {
     "gpt-4o-mini": {"input": 0.150 / 1e6, "output": 0.600 / 1e6},
-    "gpt-4o": {"input": 5.00 / 1e6, "output": 15.00 / 1e6},
+    "gpt-4o": {"input": 2.50 / 1e6, "output": 10.00 / 1e6},
     "gpt-4-turbo": {"input": 10.00 / 1e6, "output": 30.00 / 1e6},
     "gpt-4-turbo-preview": {"input": 10.00 / 1e6, "output": 30.00 / 1e6},
     "gpt-4-0125-preview": {"input": 10.00 / 1e6, "output": 30.00 / 1e6},
     "gpt-4-1106-preview": {"input": 10.00 / 1e6, "output": 30.00 / 1e6},
     "gpt-4": {"input": 30.00 / 1e6, "output": 60.00 / 1e6},
     "gpt-4-32k": {"input": 60.00 / 1e6, "output": 120.00 / 1e6},
     "gpt-3.5-turbo-1106": {"input": 1.00 / 1e6, "output": 2.00 / 1e6},
-    "gpt-3.5-turbo": {"input": 1.50 / 1e6, "output": 2.00 / 1e6},
+    "gpt-3.5-turbo": {"input": 0.50 / 1e6, "output": 1.50 / 1e6},
     "gpt-3.5-turbo-16k": {"input": 3.00 / 1e6, "output": 4.00 / 1e6},
     "gpt-3.5-turbo-0125": {"input": 0.50 / 1e6, "output": 1.50 / 1e6},
+    "gpt-3.5-turbo-instruct": {"input": 1.50 / 1e6, "output": 2.00 / 1e6},
 }
 
 default_gpt_model = "gpt-4o"
@@ -383,15 +398,20 @@ def trim_and_load_json(
 
 
 valid_multimodal_gpt_models = [
-    "gpt-4o-mini",
     "gpt-4o",
-    "gpt-4-turbo",
-    "gpt-4-turbo-preview",
+    "gpt-4o-2024-05-13",
+    "gpt-4o-2024-08-06",
+    "gpt-4o-2024-11-20",
+    "gpt-4o-mini",
+    "gpt-4o-mini-2024-07-18",
+    "gpt-4",
     "gpt-4-0125-preview",
+    "gpt-4-0613",
     "gpt-4-1106-preview",
-    "gpt-4",
+    "gpt-4-turbo",
+    "gpt-4-turbo-2024-04-09",
+    "gpt-4-turbo-preview",
     "gpt-4-32k",
-    "gpt-4-0613",
     "gpt-4-32k-0613",
 ]
 

diff --git a/deepeval/models/gpt_model_schematic.py b/deepeval/models/gpt_model_schematic.py
@@ -33,26 +33,35 @@ def log_retry_error(retry_state):
     "gpt-4o-2024-11-20",
     "gpt-4o-mini",
     "gpt-4o-mini-2024-07-18",
+    "gpt-4-32k",
+    "gpt-4-32k-0613",
 ]
 
 structured_outputs_models = [
     "gpt-4o",
-    "gpt-4o-mini"
+    "gpt-4o-2024-05-13",
+    "gpt-4o-2024-08-06",
+    "gpt-4o-2024-11-20",
+    "gpt-4o-mini",
+    "gpt-4o-mini-2024-07-18",
 ]
 
 json_mode_models = [
-    "gpt-4-turbo",
-    "gpt-4-turbo-preview",
+    "gpt-3.5-turbo",
+    "gpt-3.5-turbo-0125",
+    "gpt-3.5-turbo-1106",
+    "gpt-3.5-turbo-16k",
+    "gpt-3.5-turbo-instruct",
+    "gpt-3.5-turbo-instruct-0914",
+    "gpt-4",
     "gpt-4-0125-preview",
+    "gpt-4-0613",
     "gpt-4-1106-preview",
-    "gpt-4",
+    "gpt-4-turbo",
+    "gpt-4-turbo-2024-04-09",
+    "gpt-4-turbo-preview",
     "gpt-4-32k",
-    "gpt-4-0613",
     "gpt-4-32k-0613",
-    "gpt-3.5-turbo-1106",
-    "gpt-3.5-turbo",
-    "gpt-3.5-turbo-16k",
-    "gpt-3.5-turbo-0125",
 ]
 
 default_gpt_model = "gpt-4o"