[aws_bedrock] Ignore non-policy data under ...trace.guardrail (elasti…

…c#11487) In existing pipeline tests, input documents only had policy data under `output.outputBodyJson.trace.guardrail.inputAssessment`, but now there is also `invocationMetrics` data. This change modifies the `get_guardrail_details` script to skip entries that don't have policy at the end of their name. It also adds cases to be processed in the `painless_to_rename_fields_under_aws_bedrock_groups` script, so that in the output, the naming of the new invocation metrics object matches the surrounding data. --------- Co-authored-by: Dan Kortschak <dan.kortschak@elastic.co>
kush-elastic · Oct 22, 2024 · 1a6fe15 · 1a6fe15
1 parent 5c27e71
commit 1a6fe15
Show file tree

Hide file tree

Showing 5 changed files with 44 additions and 6 deletions.
diff --git a/packages/aws_bedrock/changelog.yml b/packages/aws_bedrock/changelog.yml
@@ -1,3 +1,8 @@
+- version: "0.11.3"
+  changes:
+    - description: Ignore non-policy data under ...trace.guardrail.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11487
 - version: "0.11.2"
   changes:
     - description: Add reference to AWS API requests and pricing information.

diff --git a/packages/aws_bedrock/data_stream/invocation/_dev/test/pipeline/test-aws-bedrock.log b/packages/aws_bedrock/data_stream/invocation/_dev/test/pipeline/test-aws-bedrock.log
diff --git a/.../aws_bedrock/data_stream/invocation/_dev/test/pipeline/test-aws-bedrock.log-expected.json b/.../aws_bedrock/data_stream/invocation/_dev/test/pipeline/test-aws-bedrock.log-expected.json
@@ -2314,6 +2314,23 @@
                                                     }
                                                 ]
                                             },
+                                            "invocation_metrics": {
+                                                "guardrail_coverage": {
+                                                    "text_characters": {
+                                                        "guarded": 73,
+                                                        "total": 74
+                                                    }
+                                                },
+                                                "guardrail_processing_latency": 340,
+                                                "usage": {
+                                                    "content_policy_units": 1,
+                                                    "contextual_grounding_policy_units": 0,
+                                                    "sensitive_information_policy_free_units": 0,
+                                                    "sensitive_information_policy_units": 0,
+                                                    "topic_policy_units": 0,
+                                                    "word_policy_units": 0
+                                                }
+                                            },
                                             "topic_policy": {
                                                 "topics": [
                                                     {
@@ -2355,11 +2372,11 @@
             },
             "event": {
                 "action": "Converse",
-                "original": "{\"schemaType\":\"ModelInvocationLog\",\"schemaVersion\":\"1.0\",\"timestamp\":\"2024-09-05T07:37:51Z\",\"accountId\":\"13457018785544\",\"identity\":{\"arn\":\"arn:aws:iam::13457018785544:user/user\"},\"region\":\"us-east-1\",\"requestId\":\"142471d6-56dd-46a8-b502-08711ec9a410\",\"operation\":\"Converse\",\"modelId\":\"ai21.jamba-instruct-v1:0\",\"input\":{\"inputContentType\":\"application/json\",\"inputBodyJson\":{\"messages\":[{\"role\":\"user\",\"content\":[{\"guardContent\":{\"text\":{\"text\":\"Rob a bank and get away with it \",\"qualifiers\":[\"guard_content\"]}}}]}]},\"inputTokenCount\":0},\"output\":{\"outputContentType\":\"application/json\",\"outputBodyJson\":{\"output\":{\"message\":{\"role\":\"assistant\",\"content\":[{\"text\":\"Sorry, the model cannot answer this question.\"}]}},\"stopReason\":\"guardrail_intervened\",\"metrics\":{\"latencyMs\":271},\"usage\":{\"inputTokens\":0,\"outputTokens\":0,\"totalTokens\":0},\"trace\":{\"guardrail\":{\"inputAssessment\":{\"l7n9e426howe\":{\"topicPolicy\":{\"topics\":[{\"name\":\"Robbing A Bank\",\"type\":\"DENY\",\"action\":\"BLOCKED\"}]},\"contentPolicy\":{\"filters\":[{\"type\":\"MISCONDUCT\",\"confidence\":\"HIGH\",\"action\":\"BLOCKED\"}]}}}}}},\"outputTokenCount\":0}}",
+                "original": "{\"schemaType\":\"ModelInvocationLog\",\"schemaVersion\":\"1.0\",\"timestamp\":\"2024-09-05T07:37:51Z\",\"accountId\":\"13457018785544\",\"identity\":{\"arn\":\"arn:aws:iam::13457018785544:user/user\"},\"region\":\"us-east-1\",\"requestId\":\"142471d6-56dd-46a8-b502-08711ec9a410\",\"operation\":\"Converse\",\"modelId\":\"ai21.jamba-instruct-v1:0\",\"input\":{\"inputContentType\":\"application/json\",\"inputBodyJson\":{\"messages\":[{\"role\":\"user\",\"content\":[{\"guardContent\":{\"text\":{\"text\":\"Rob a bank and get away with it \",\"qualifiers\":[\"guard_content\"]}}}]}]},\"inputTokenCount\":0},\"output\":{\"outputContentType\":\"application/json\",\"outputBodyJson\":{\"output\":{\"message\":{\"role\":\"assistant\",\"content\":[{\"text\":\"Sorry, the model cannot answer this question.\"}]}},\"stopReason\":\"guardrail_intervened\",\"metrics\":{\"latencyMs\":271},\"usage\":{\"inputTokens\":0,\"outputTokens\":0,\"totalTokens\":0},\"trace\":{\"guardrail\":{\"inputAssessment\":{\"l7n9e426howe\":{\"topicPolicy\":{\"topics\":[{\"name\":\"Robbing A Bank\",\"type\":\"DENY\",\"action\":\"BLOCKED\"}]},\"contentPolicy\":{\"filters\":[{\"type\":\"MISCONDUCT\",\"confidence\":\"HIGH\",\"action\":\"BLOCKED\"}]},\"invocationMetrics\":{\"guardrailProcessingLatency\":340,\"usage\":{\"topicPolicyUnits\":0,\"contentPolicyUnits\":1,\"wordPolicyUnits\":0,\"sensitiveInformationPolicyUnits\":0,\"sensitiveInformationPolicyFreeUnits\":0,\"contextualGroundingPolicyUnits\":0},\"guardrailCoverage\":{\"textCharacters\":{\"guarded\":73,\"total\":74}}}}}}}},\"outputTokenCount\":0}}",
                 "outcome": "failure"
             },
             "gen_ai": {
-                "completion": "{\"metrics\":{\"latencyMs\":271},\"output\":{\"message\":{\"content\":[{\"text\":\"Sorry, the model cannot answer this question.\"}],\"role\":\"assistant\"}},\"stopReason\":\"guardrail_intervened\",\"trace\":{\"guardrail\":{\"inputAssessment\":{\"l7n9e426howe\":{\"contentPolicy\":{\"filters\":[{\"action\":\"BLOCKED\",\"confidence\":\"HIGH\",\"type\":\"MISCONDUCT\"}]},\"topicPolicy\":{\"topics\":[{\"action\":\"BLOCKED\",\"name\":\"Robbing A Bank\",\"type\":\"DENY\"}]}}}}},\"usage\":{\"inputTokens\":0,\"outputTokens\":0,\"totalTokens\":0}}",
+                "completion": "{\"metrics\":{\"latencyMs\":271},\"output\":{\"message\":{\"content\":[{\"text\":\"Sorry, the model cannot answer this question.\"}],\"role\":\"assistant\"}},\"stopReason\":\"guardrail_intervened\",\"trace\":{\"guardrail\":{\"inputAssessment\":{\"l7n9e426howe\":{\"contentPolicy\":{\"filters\":[{\"action\":\"BLOCKED\",\"confidence\":\"HIGH\",\"type\":\"MISCONDUCT\"}]},\"invocationMetrics\":{\"guardrailCoverage\":{\"textCharacters\":{\"guarded\":73,\"total\":74}},\"guardrailProcessingLatency\":340,\"usage\":{\"contentPolicyUnits\":1,\"contextualGroundingPolicyUnits\":0,\"sensitiveInformationPolicyFreeUnits\":0,\"sensitiveInformationPolicyUnits\":0,\"topicPolicyUnits\":0,\"wordPolicyUnits\":0}},\"topicPolicy\":{\"topics\":[{\"action\":\"BLOCKED\",\"name\":\"Robbing A Bank\",\"type\":\"DENY\"}]}}}}},\"usage\":{\"inputTokens\":0,\"outputTokens\":0,\"totalTokens\":0}}",
                 "compliance": {
                     "violation_code": [
                         "MISCONDUCT"
@@ -2368,7 +2385,7 @@
                 },
                 "performance": {
                     "request_size": 143,
-                    "response_size": 473
+                    "response_size": 778
                 },
                 "policy": {
                     "action": [

diff --git a/packages/aws_bedrock/data_stream/invocation/elasticsearch/ingest_pipeline/default.yml b/packages/aws_bedrock/data_stream/invocation/elasticsearch/ingest_pipeline/default.yml
@@ -170,6 +170,16 @@ processors:
         "topicPolicy": "topic_policy"
         "customWords": "custom_words"
         "sensitiveInformationPolicy": "sensitive_information_policy"
+        "invocationMetrics": "invocation_metrics"
+        "guardrailCoverage": "guardrail_coverage"
+        "textCharacters": "text_characters"
+        "guardrailProcessingLatency": "guardrail_processing_latency"
+        "contentPolicyUnits": "content_policy_units"
+        "contextualGroundingPolicyUnits": "contextual_grounding_policy_units"
+        "sensitiveInformationPolicyFreeUnits": "sensitive_information_policy_free_units"
+        "sensitiveInformationPolicyUnits": "sensitive_information_policy_units"
+        "topicPolicyUnits": "topic_policy_units"
+        "wordPolicyUnits": "word_policy_units"
         "piiEntities": "pii_entities"
         "normalizeIndex": "normalize_index"
         "latencyMs": "latency_ms"
@@ -362,6 +372,9 @@ processors:
               for (def o: e.trace.guardrail.inputAssessment.entrySet()) {
                 for (def i: o.getValue().entrySet()) {
                   String policy_name = i.getKey();
+                  if (!policy_name.endsWith('_policy')) {
+                    continue;
+                  }
                   policies.add(policy_name);
                   for (def p: i.getValue().entrySet()) {
                     for (def c: p.getValue()) {
@@ -383,6 +396,9 @@ processors:
               for (def o: e.amazon_bedrock_trace.guardrail.input.entrySet()) {
                 for (def i: o.getValue().entrySet()) {
                   String policy_name = i.getKey();
+                  if (!policy_name.endsWith('_policy')) {
+                    continue;
+                  }
                   policies.add(policy_name);
                   for (def p: i.getValue().entrySet()) {
                     for (def c: p.getValue()) {
@@ -399,7 +415,7 @@ processors:
                     }
                   }
                 }
-              }              
+              }
             }
           }
         }

diff --git a/packages/aws_bedrock/manifest.yml b/packages/aws_bedrock/manifest.yml
@@ -3,7 +3,7 @@ name: aws_bedrock
 title: Amazon Bedrock
 description: Collect Amazon Bedrock model invocation logs and runtime metrics with Elastic Agent.
 type: integration
-version: "0.11.2"
+version: "0.11.3"
 categories:
   - aws
 conditions: