Merge pull request #1296 from kritinv/telemetry-new-features

feature status telemetry
confident-ai · Jan 22, 2025 · 3a7ab47 · 3a7ab47
2 parents 4f3af89 + 3142af0
commit 3a7ab47
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 1 deletion.
diff --git a/deepeval/telemetry.py b/deepeval/telemetry.py
@@ -131,6 +131,7 @@ def capture_evaluation_run(type: str):
             span.set_attribute("environment", IS_RUNNING_IN_JUPYTER)
             span.set_attribute("user.status", get_status())
             span.set_attribute("user.unique_id", get_unique_id())
+            span.set_attribute("feature_status.evaluation", get_feature_status(Feature.EVALUATION))
             if anonymous_public_ip:
                 span.set_attribute("user.public_ip", anonymous_public_ip)
             set_last_feature(Feature.EVALUATION)
@@ -164,6 +165,7 @@ def capture_synthesizer_run(
             span.set_attribute("environment", IS_RUNNING_IN_JUPYTER)
             span.set_attribute("user.status", get_status())
             span.set_attribute("user.unique_id", get_unique_id())
+            span.set_attribute("feature_status.synthesizer", get_feature_status(Feature.SYNTHESIZER))
             span.set_attribute("method", method)
             span.set_attribute("max_generations", max_generations)
             span.set_attribute("evolutions", num_evolutions)
@@ -188,6 +190,7 @@ def capture_red_teamer_run(
             span.set_attribute("environment", IS_RUNNING_IN_JUPYTER)
             span.set_attribute("user.status", get_status())
             span.set_attribute("user.unique_id", get_unique_id())
+            span.set_attribute("feature_status.redteaming", get_feature_status(Feature.REDTEAMING))
             span.set_attribute(
                 "attacks_per_vulnerability", attacks_per_vulnerability_type
             )
@@ -213,6 +216,7 @@ def capture_guardrails(guards: List[str]):
             span.set_attribute("environment", IS_RUNNING_IN_JUPYTER)
             span.set_attribute("user.status", get_status())
             span.set_attribute("user.unique_id", get_unique_id())
+            span.set_attribute("feature_status.guardrail", get_feature_status(Feature.GUARDRAIL))
             for guard in guards:
                 span.set_attribute(f"vulnerability.{guard}", 1)
             set_last_feature(Feature.GUARDRAIL)
@@ -230,6 +234,7 @@ def capture_benchmark_run(benchmark: str, num_tasks: int):
             span.set_attribute("environment", IS_RUNNING_IN_JUPYTER)
             span.set_attribute("user.status", get_status())
             span.set_attribute("user.unique_id", get_unique_id())
+            span.set_attribute("feature_status.benchmark", get_feature_status(Feature.BENCHMARK))
             span.set_attribute("benchmark", benchmark)
             span.set_attribute("num_tasks", num_tasks)
             set_last_feature(Feature.BENCHMARK)
@@ -315,4 +320,12 @@ def set_last_feature(feature: Feature):
         raise ValueError(f"Invalid feature: {feature}")
     data = read_telemetry_file()
     data["DEEPEVAL_LAST_FEATURE"] = feature.value
+    feature_status_key = f"DEEPEVAL_{feature.value.upper()}_STATUS"
+    data[feature_status_key] = "old"
     write_telemetry_file(data)
+
+def get_feature_status(feature: Feature) -> str:
+    """Gets the status of a feature ('new' or 'old') from the telemetry file."""
+    data = read_telemetry_file()
+    feature_status_key = f"DEEPEVAL_{feature.value.upper()}_STATUS"
+    return data.get(feature_status_key, "new")
diff --git a/tests/test_synthesizer.py b/tests/test_synthesizer.py
@@ -285,7 +285,7 @@ def test_generate_goldens_from_docs(synthesizer: Synthesizer):
 )
 synthesizer_async = Synthesizer(async_mode=True, max_concurrent=3)
 
-test_generate_goldens_from_docs(synthesizer_sync)
+# test_generate_goldens_from_docs(synthesizer_sync)
 test_generate_goldens_from_docs(synthesizer_async)
 
 #########################################################