Merge branch 'main' into topic/openai-support

deepeval/__init__.py

@@ -1,3 +1,4 @@
+import os
 import warnings
 import re
 
@@ -65,4 +66,9 @@ def check_for_update():
         pass
 
 
-check_for_update()
+def update_warning_opt_out():
+    return os.getenv("DEEPEVAL_UPDATE_WARNING_OPT_OUT") == "YES"
+
+
+if not update_warning_opt_out():
+    check_for_update()

deepeval/_version.py

@@ -1 +1 @@
-__version__: str = "2.1.7"
+__version__: str = "2.1.8"

deepeval/guardrails/guardrails.py

@@ -47,8 +47,6 @@ def guard_input(self, input: str):
             )
             body = api_guardrails.model_dump(by_alias=True, exclude_none=True)
 
-            print(body)
-
             # API request
             if is_confident():
                 api = Api(base_url=BASE_URL)
@@ -60,7 +58,7 @@ def guard_input(self, input: str):
                 return GuardsResponseData(**response).result
             else:
                 raise Exception(
-                    "Access denied: You need Enterprise access on Confident AI to use deepeval's guardrails."
+                    "Access denied: You need Premium access on Confident AI to use deepeval's guardrails."
                 )
 
     def guard_output(self, input: str, response: str):
@@ -94,7 +92,6 @@ def guard_output(self, input: str, response: str):
             # API request
             if is_confident():
                 api = Api(base_url=BASE_URL)
-                print("!!!!!")
                 response = api.send_request(
                     method=HttpMethods.POST,
                     endpoint=Endpoints.GUARDRAILS_ENDPOINT,
@@ -104,5 +101,5 @@ def guard_output(self, input: str, response: str):
                 return GuardsResponseData(**response).result
             else:
                 raise Exception(
-                    "Access denied: You need Enterprise access on Confident AI to use deepeval's guardrails."
+                    "Access denied: You need Premium access on Confident AI to use deepeval's guardrails."
                 )

deepeval/metrics/answer_relevancy/answer_relevancy.py

@@ -127,10 +127,9 @@ async def _a_generate_reason(self, input: str) -> str:
             score=format(self.score, ".2f"),
         )
         if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
+            res, cost = await self.model.a_generate(prompt, schema=Reason)
             self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return data["reason"]
+            return res.reason
         else:
             try:
                 res: Reason = await self.model.a_generate(
@@ -158,10 +157,9 @@ def _generate_reason(self, input: str) -> str:
         )
 
         if self.using_native_model:
-            res, cost = self.model.generate(prompt)
+            res, cost = self.model.generate(prompt, schema=Reason)
             self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return data["reason"]
+            return res.reason
         else:
             try:
                 res: Reason = self.model.generate(prompt=prompt, schema=Reason)
@@ -182,10 +180,9 @@ async def _a_generate_verdicts(
             actual_output=self.statements,
         )
         if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
+            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
             self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return [AnswerRelvancyVerdict(**item) for item in data["verdicts"]]
+            return [item for item in res.verdicts]
         else:
             try:
                 res: Verdicts = await self.model.a_generate(
@@ -208,10 +205,9 @@ def _generate_verdicts(self, input: str) -> List[AnswerRelvancyVerdict]:
             actual_output=self.statements,
         )
         if self.using_native_model:
-            res, cost = self.model.generate(prompt)
+            res, cost = self.model.generate(prompt, schema=Verdicts)
             self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return [AnswerRelvancyVerdict(**item) for item in data["verdicts"]]
+            return [item for item in res.verdicts]
         else:
             try:
                 res: Verdicts = self.model.generate(prompt, schema=Verdicts)
@@ -231,10 +227,9 @@ async def _a_generate_statements(
             actual_output=actual_output,
         )
         if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
+            res, cost = await self.model.a_generate(prompt, schema=Statements)
             self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return data["statements"]
+            return res.statements
         else:
             try:
                 res: Statements = await self.model.a_generate(
@@ -254,10 +249,9 @@ def _generate_statements(
             actual_output=actual_output,
         )
         if self.using_native_model:
-            res, cost = self.model.generate(prompt)
+            res, cost = self.model.generate(prompt, schema=Statements)
             self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return data["statements"]
+            return res.statements
         else:
             try:
                 res: Statements = self.model.generate(prompt, schema=Statements)

deepeval/metrics/bias/bias.py

@@ -125,10 +125,9 @@ async def _a_generate_reason(self) -> str:
         )
 
         if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
+            res, cost = await self.model.a_generate(prompt, schema=Reason)
             self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return data["reason"]
+            return res.reason
         else:
             try:
                 res: Reason = await self.model.a_generate(prompt, schema=Reason)
@@ -153,10 +152,9 @@ def _generate_reason(self) -> str:
         )
 
         if self.using_native_model:
-            res, cost = self.model.generate(prompt)
+            res, cost = self.model.generate(prompt, schema=Reason)
             self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return data["reason"]
+            return res.reason
         else:
             try:
                 res: Reason = self.model.generate(prompt, schema=Reason)
@@ -173,10 +171,9 @@ async def _a_generate_verdicts(self) -> List[BiasVerdict]:
         verdicts: List[BiasVerdict] = []
         prompt = BiasTemplate.generate_verdicts(opinions=self.opinions)
         if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
+            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
             self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            verdicts = [BiasVerdict(**item) for item in data["verdicts"]]
+            verdicts = [item for item in res.verdicts]
             return verdicts
         else:
             try:
@@ -198,10 +195,9 @@ def _generate_verdicts(self) -> List[BiasVerdict]:
         verdicts: List[BiasVerdict] = []
         prompt = BiasTemplate.generate_verdicts(opinions=self.opinions)
         if self.using_native_model:
-            res, cost = self.model.generate(prompt)
+            res, cost = self.model.generate(prompt, schema=BiasVerdict)
             self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            verdicts = [BiasVerdict(**item) for item in data["verdicts"]]
+            verdicts = [item for item in res.verdicts]
             return verdicts
         else:
             try:
@@ -217,10 +213,9 @@ def _generate_verdicts(self) -> List[BiasVerdict]:
     async def _a_generate_opinions(self, actual_output: str) -> List[str]:
         prompt = BiasTemplate.generate_opinions(actual_output=actual_output)
         if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
+            res, cost = await self.model.a_generate(prompt, schema=Opinions)
             self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return data["opinions"]
+            return res.opinions
         else:
             try:
                 res: Opinions = await self.model.a_generate(
@@ -235,10 +230,9 @@ async def _a_generate_opinions(self, actual_output: str) -> List[str]:
     def _generate_opinions(self, actual_output: str) -> List[str]:
         prompt = BiasTemplate.generate_opinions(actual_output=actual_output)
         if self.using_native_model:
-            res, cost = self.model.generate(prompt)
+            res, cost = self.model.generate(prompt, schema=Opinions)
             self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return data["opinions"]
+            return res.opinions
         else:
             try:
                 res: Opinions = self.model.generate(prompt, schema=Opinions)

deepeval/metrics/contextual_precision/contextual_precision.py

@@ -133,10 +133,9 @@ async def _a_generate_reason(self, input: str):
         )
 
         if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
+            res, cost = await self.model.a_generate(prompt, schema=Reason)
             self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return data["reason"]
+            return res.reason
         else:
             try:
                 res: Reason = await self.model.a_generate(prompt, schema=Reason)
@@ -161,10 +160,9 @@ def _generate_reason(self, input: str):
         )
 
         if self.using_native_model:
-            res, cost = self.model.generate(prompt)
+            res, cost = self.model.generate(prompt, schema=Reason)
             self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return data["reason"]
+            return res.reason
         else:
             try:
                 res: Reason = self.model.generate(prompt, schema=Reason)
@@ -183,12 +181,9 @@ async def _a_generate_verdicts(
             retrieval_context=retrieval_context,
         )
         if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
+            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
             self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            verdicts = [
-                ContextualPrecisionVerdict(**item) for item in data["verdicts"]
-            ]
+            verdicts = [item for item in res.verdicts]
             return verdicts
         else:
             try:
@@ -215,12 +210,9 @@ def _generate_verdicts(
             retrieval_context=retrieval_context,
         )
         if self.using_native_model:
-            res, cost = self.model.generate(prompt)
+            res, cost = self.model.generate(prompt, schema=Verdicts)
             self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            verdicts = [
-                ContextualPrecisionVerdict(**item) for item in data["verdicts"]
-            ]
+            verdicts = [item for item in res.verdicts]
             return verdicts
         else:
             try:

deepeval/metrics/contextual_recall/contextual_recall.py

@@ -134,10 +134,9 @@ async def _a_generate_reason(self, expected_output: str):
         )
 
         if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
+            res, cost = await self.model.a_generate(prompt, schema=Reason)
             self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return data["reason"]
+            return res.reason
         else:
             try:
                 res: Reason = await self.model.a_generate(prompt, schema=Reason)
@@ -167,10 +166,9 @@ def _generate_reason(self, expected_output: str):
         )
 
         if self.using_native_model:
-            res, cost = self.model.generate(prompt)
+            res, cost = self.model.generate(prompt, schema=Reason)
             self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return data["reason"]
+            return res.reason
         else:
             try:
                 res: Reason = self.model.generate(prompt, schema=Reason)
@@ -200,12 +198,9 @@ async def _a_generate_verdicts(
             expected_output=expected_output, retrieval_context=retrieval_context
         )
         if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
+            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
             self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            verdicts = [
-                ContextualRecallVerdict(**item) for item in data["verdicts"]
-            ]
+            verdicts = [item for item in res.verdicts]
             return verdicts
         else:
             try:
@@ -229,12 +224,9 @@ def _generate_verdicts(
             expected_output=expected_output, retrieval_context=retrieval_context
         )
         if self.using_native_model:
-            res, cost = self.model.generate(prompt)
+            res, cost = self.model.generate(prompt, schema=Verdicts)
             self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            verdicts = [
-                ContextualRecallVerdict(**item) for item in data["verdicts"]
-            ]
+            verdicts = [item for item in res.verdicts]
             return verdicts
         else:
             try:

deepeval/metrics/contextual_relevancy/contextual_relevancy.py

@@ -136,10 +136,9 @@ async def _a_generate_reason(self, input: str):
             score=format(self.score, ".2f"),
         )
         if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
+            res, cost = await self.model.a_generate(prompt, schema=Reason)
             self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return data["reason"]
+            return res.reason
         else:
             try:
                 res: Reason = await self.model.a_generate(prompt, schema=Reason)
@@ -169,10 +168,9 @@ def _generate_reason(self, input: str):
             score=format(self.score, ".2f"),
         )
         if self.using_native_model:
-            res, cost = self.model.generate(prompt)
+            res, cost = self.model.generate(prompt, schema=Reason)
             self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return data["reason"]
+            return res.reason
         else:
             try:
                 res: Reason = self.model.generate(prompt, schema=Reason)
@@ -204,10 +202,9 @@ async def _a_generate_verdicts(
             input=input, context=context
         )
         if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt)
+            res, cost = await self.model.a_generate(prompt, schema=ContextualRelevancyVerdicts)
             self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return ContextualRelevancyVerdicts(**data)
+            return res
         else:
             try:
                 res = await self.model.a_generate(
@@ -226,10 +223,9 @@ def _generate_verdicts(
             input=input, context=context
         )
         if self.using_native_model:
-            res, cost = self.model.generate(prompt)
+            res, cost = self.model.generate(prompt, schema=ContextualRelevancyVerdicts)
             self.evaluation_cost += cost
-            data = trimAndLoadJson(res, self)
-            return ContextualRelevancyVerdicts(**data)
+            return res
         else:
             try:
                 res = self.model.generate(