e2e: add agent-tool test

.github/terraform/aws.tf

@@ -1,5 +1,7 @@
 provider "aws" {
-  region = "us-east-1"
+#   region = "us-east-1"
+  region = "eu-central-1"
+
 }
 
 resource "aws_iam_user" "envoy_ai_gateway_ci_user" {

examples/basic/basic.yaml

@@ -111,7 +111,8 @@ metadata:
 spec:
   type: AWSCredentials
   awsCredentials:
-    region: us-east-1
+#    region: us-east-1
+    region: eu-central-1
     credentialsFile:
       secretRef:
         name: envoy-ai-gateway-basic-aws-credentials
@@ -135,7 +136,8 @@ metadata:
 spec:
   endpoints:
     - fqdn:
-        hostname: bedrock-runtime.us-east-1.amazonaws.com
+#        hostname: bedrock-runtime.us-east-1.amazonaws.com
+        hostname: bedrock-runtime.eu-central-1.amazonaws.com
         port: 443
 ---
 apiVersion: gateway.networking.k8s.io/v1alpha3
@@ -164,7 +166,8 @@ spec:
       name: envoy-ai-gateway-basic-aws
   validation:
     wellKnownCACertificates: "System"
-    hostname: bedrock-runtime.us-east-1.amazonaws.com
+#    hostname: bedrock-runtime.us-east-1.amazonaws.com
+    hostname: bedrock-runtime.eu-central-1.amazonaws.com
 ---
 apiVersion: v1
 kind: Secret

filterapi/filterconfig_test.go

@@ -89,7 +89,8 @@ rules:
 	require.Equal(t, "OpenAI", string(cfg.Rules[1].Backends[0].Schema.Name))
 	require.Equal(t, "apikey.txt", cfg.Rules[0].Backends[0].Auth.APIKey.Filename)
 	require.Equal(t, "aws.txt", cfg.Rules[0].Backends[1].Auth.AWSAuth.CredentialFileName)
-	require.Equal(t, "us-east-1", cfg.Rules[0].Backends[1].Auth.AWSAuth.Region)
+	//require.Equal(t, "us-east-1", cfg.Rules[0].Backends[1].Auth.AWSAuth.Region)
+	require.Equal(t, "eu-central-1", cfg.Rules[0].Backends[1].Auth.AWSAuth.Region)
 
 	t.Run("not found", func(t *testing.T) {
 		_, err := filterapi.UnmarshalConfigYaml("not-found.yaml")

internal/extproc/translator/openai_awsbedrock.go

@@ -311,16 +311,35 @@ func (o *openAIToAWSBedrockTranslatorV1ChatCompletion) openAIMessageToBedrockMes
 func (o *openAIToAWSBedrockTranslatorV1ChatCompletion) openAIMessageToBedrockMessageRoleTool(
 	openAiMessage *openai.ChatCompletionToolMessageParam, role string,
 ) (*awsbedrock.Message, error) {
+	var content []*awsbedrock.ToolResultContentBlock
+
+	switch v := openAiMessage.Content.Value.(type) {
+	case string:
+		content = []*awsbedrock.ToolResultContentBlock{
+			{
+				Text: &v,
+			},
+		}
+	case []openai.ChatCompletionContentPartTextParam:
+		var combinedText string
+		for _, part := range v {
+			combinedText += part.Text
+		}
+		content = []*awsbedrock.ToolResultContentBlock{
+			{
+				Text: &combinedText,
+			},
+		}
+	default:
+		return nil, fmt.Errorf("unexpected content type for tool message: %T", openAiMessage.Content.Value)
+	}
+
 	return &awsbedrock.Message{
 		Role: role,
 		Content: []*awsbedrock.ContentBlock{
 			{
 				ToolResult: &awsbedrock.ToolResultBlock{
-					Content: []*awsbedrock.ToolResultContentBlock{
-						{
-							Text: openAiMessage.Content.Value.(*string),
-						},
-					},
+					Content: content,
 				},
 			},
 		},
@@ -481,6 +500,7 @@ func (o *openAIToAWSBedrockTranslatorV1ChatCompletion) ResponseError(respHeaders
 	} else {
 		var buf []byte
 		buf, err = io.ReadAll(body)
+		fmt.Printf("\nprinting body from ResponseError:\n %v\n", string(buf))
 		if err != nil {
 			return nil, nil, fmt.Errorf("failed to read error body: %w", err)
 		}
@@ -548,6 +568,7 @@ func (o *openAIToAWSBedrockTranslatorV1ChatCompletion) ResponseBody(respHeaders
 			mut.Body = append(mut.Body, oaiEventBytes...)
 			mut.Body = append(mut.Body, []byte("\n\n")...)
 		}
+		fmt.Printf("\nprinting mut.Body %v", string(mut.Body))
 
 		if endOfStream {
 			mut.Body = append(mut.Body, []byte("data: [DONE]\n")...)
@@ -559,6 +580,7 @@ func (o *openAIToAWSBedrockTranslatorV1ChatCompletion) ResponseBody(respHeaders
 	if err = json.NewDecoder(body).Decode(&bedrockResp); err != nil {
 		return nil, nil, tokenUsage, fmt.Errorf("failed to unmarshal body: %w", err)
 	}
+	fmt.Printf("\nbedrock output message from converse: %v\n", len(bedrockResp.Output.Message.Content))
 
 	openAIResp := openai.ChatCompletionResponse{
 		Object:  "chat.completion",
@@ -577,18 +599,41 @@ func (o *openAIToAWSBedrockTranslatorV1ChatCompletion) ResponseBody(respHeaders
 			CompletionTokens: bedrockResp.Usage.OutputTokens,
 		}
 	}
-	for i, output := range bedrockResp.Output.Message.Content {
+
+	// Merge bedrock response content into openai response choices
+	for i := 0; i < len(bedrockResp.Output.Message.Content); i++ {
+		output := bedrockResp.Output.Message.Content[i]
 		choice := openai.ChatCompletionResponseChoice{
 			Index: (int64)(i),
 			Message: openai.ChatCompletionResponseChoiceMessage{
-				Content: output.Text,
-				Role:    bedrockResp.Output.Message.Role,
+				Role: bedrockResp.Output.Message.Role,
 			},
 			FinishReason: o.bedrockStopReasonToOpenAIStopReason(bedrockResp.StopReason),
 		}
-		if toolCall := o.bedrockToolUseToOpenAICalls(output.ToolUse); toolCall != nil {
-			choice.Message.ToolCalls = []openai.ChatCompletionMessageToolCallParam{*toolCall}
+
+		if output.Text != nil {
+			choice.Message.Content = output.Text
+		}
+
+		if output.ToolUse != nil {
+			if toolCall := o.bedrockToolUseToOpenAICalls(output.ToolUse); toolCall != nil {
+				choice.Message.ToolCalls = []openai.ChatCompletionMessageToolCallParam{*toolCall}
+			}
+		}
+
+		// Check if the next element should be merged -
+		// A model may return the tool config in a separate message, 
+		// the message text + tool config should be merged for the openai responsed
+		if i+1 < len(bedrockResp.Output.Message.Content) {
+			nextOutput := bedrockResp.Output.Message.Content[i+1]
+			if nextOutput.Text == nil && nextOutput.ToolUse != nil {
+				if toolCall := o.bedrockToolUseToOpenAICalls(nextOutput.ToolUse); toolCall != nil {
+					choice.Message.ToolCalls = append(choice.Message.ToolCalls, *toolCall)
+				}
+				i++ // Skip the next element as it has been merged
+			}
 		}
+
 		openAIResp.Choices = append(openAIResp.Choices, choice)
 	}
 

internal/extproc/translator/openai_awsbedrock_test.go

@@ -1233,3 +1233,210 @@ func TestOpenAIToAWSBedrockTranslator_convertEvent(t *testing.T) {
 		})
 	}
 }
+
+func TestOpenAIToAWSBedrockTranslatorV1ChatCompletion_ResponseBody_MergeContent(t *testing.T) {
+	o := &openAIToAWSBedrockTranslatorV1ChatCompletion{}
+	bedrockResp := awsbedrock.ConverseResponse{
+		Usage: &awsbedrock.TokenUsage{
+			InputTokens:  10,
+			OutputTokens: 20,
+			TotalTokens:  30,
+		},
+		Output: &awsbedrock.ConverseOutput{
+			Message: awsbedrock.Message{
+				Role: "assistant",
+				Content: []*awsbedrock.ContentBlock{
+					{Text: ptr.To("response")},
+					{ToolUse: &awsbedrock.ToolUseBlock{
+						Name:      "exec_python_code",
+						ToolUseID: "call_6g7a",
+						Input:     map[string]interface{}{"code_block": "from playwright.sync_api import sync_playwright\n"},
+					}},
+				},
+			},
+		},
+	}
+
+	body, err := json.Marshal(bedrockResp)
+	require.NoError(t, err)
+
+	hm, bm, usedToken, err := o.ResponseBody(nil, bytes.NewBuffer(body), false)
+	require.NoError(t, err)
+	require.NotNil(t, bm)
+	require.NotNil(t, bm.Mutation)
+	require.NotNil(t, bm.Mutation.(*extprocv3.BodyMutation_Body))
+	newBody := bm.Mutation.(*extprocv3.BodyMutation_Body).Body
+	require.NotNil(t, newBody)
+	require.NotNil(t, hm)
+	require.NotNil(t, hm.SetHeaders)
+	require.Len(t, hm.SetHeaders, 1)
+	require.Equal(t, "content-length", hm.SetHeaders[0].Header.Key)
+	require.Equal(t, strconv.Itoa(len(newBody)), string(hm.SetHeaders[0].Header.RawValue))
+
+	var openAIResp openai.ChatCompletionResponse
+	err = json.Unmarshal(newBody, &openAIResp)
+	require.NoError(t, err)
+
+	expectedResponse := openai.ChatCompletionResponse{
+		Object: "chat.completion",
+		Usage: openai.ChatCompletionResponseUsage{
+			TotalTokens:      30,
+			PromptTokens:     10,
+			CompletionTokens: 20,
+		},
+		Choices: []openai.ChatCompletionResponseChoice{
+			{
+				Index: 0,
+				Message: openai.ChatCompletionResponseChoiceMessage{
+					Content: ptr.To("response"),
+					Role:    "assistant",
+					ToolCalls: []openai.ChatCompletionMessageToolCallParam{
+						{
+							ID: "call_6g7a",
+							Function: openai.ChatCompletionMessageToolCallFunctionParam{
+								Name:      "exec_python_code",
+								Arguments: "{\"code_block\":\"from playwright.sync_api import sync_playwright\\n\"}",
+							},
+							Type: openai.ChatCompletionMessageToolCallTypeFunction,
+						},
+					},
+				},
+				FinishReason: openai.ChatCompletionChoicesFinishReasonStop,
+			},
+		},
+	}
+
+	require.Equal(t,
+		LLMTokenUsage{
+			InputTokens:  uint32(expectedResponse.Usage.PromptTokens),     //nolint:gosec
+			OutputTokens: uint32(expectedResponse.Usage.CompletionTokens), //nolint:gosec
+			TotalTokens:  uint32(expectedResponse.Usage.TotalTokens),      //nolint:gosec
+		}, usedToken)
+	if !cmp.Equal(openAIResp, expectedResponse) {
+		t.Errorf("ResponseBody(), diff(got, expected) = %s\n", cmp.Diff(openAIResp, expectedResponse))
+	}
+}
+
+func TestOpenAIToAWSBedrockTranslatorV1ChatCompletion_ResponseBody_HandleContentTypes(t *testing.T) {
+	o := &openAIToAWSBedrockTranslatorV1ChatCompletion{}
+	tests := []struct {
+		name           string
+		bedrockResp    awsbedrock.ConverseResponse
+		expectedOutput openai.ChatCompletionResponse
+	}{
+		{
+			name: "content as string",
+			bedrockResp: awsbedrock.ConverseResponse{
+				Usage: &awsbedrock.TokenUsage{
+					InputTokens:  10,
+					OutputTokens: 20,
+					TotalTokens:  30,
+				},
+				Output: &awsbedrock.ConverseOutput{
+					Message: awsbedrock.Message{
+						Role: "assistant",
+						Content: []*awsbedrock.ContentBlock{
+							{Text: ptr.To("response")},
+						},
+					},
+				},
+			},
+			expectedOutput: openai.ChatCompletionResponse{
+				Object: "chat.completion",
+				Usage: openai.ChatCompletionResponseUsage{
+					TotalTokens:      30,
+					PromptTokens:     10,
+					CompletionTokens: 20,
+				},
+				Choices: []openai.ChatCompletionResponseChoice{
+					{
+						Index: 0,
+						Message: openai.ChatCompletionResponseChoiceMessage{
+							Content: ptr.To("response"),
+							Role:    "assistant",
+						},
+						FinishReason: openai.ChatCompletionChoicesFinishReasonStop,
+					},
+				},
+			},
+		},
+		{
+			name: "content as array",
+			bedrockResp: awsbedrock.ConverseResponse{
+				Usage: &awsbedrock.TokenUsage{
+					InputTokens:  10,
+					OutputTokens: 20,
+					TotalTokens:  30,
+				},
+				Output: &awsbedrock.ConverseOutput{
+					Message: awsbedrock.Message{
+						Role: "assistant",
+						Content: []*awsbedrock.ContentBlock{
+							{Text: ptr.To("response part 1")},
+							{Text: ptr.To("response part 2")},
+						},
+					},
+				},
+			},
+			expectedOutput: openai.ChatCompletionResponse{
+				Object: "chat.completion",
+				Usage: openai.ChatCompletionResponseUsage{
+					TotalTokens:      30,
+					PromptTokens:     10,
+					CompletionTokens: 20,
+				},
+				Choices: []openai.ChatCompletionResponseChoice{
+					{
+						Index: 0,
+						Message: openai.ChatCompletionResponseChoiceMessage{
+							Content: ptr.To("response part 1"),
+							Role:    "assistant",
+						},
+						FinishReason: openai.ChatCompletionChoicesFinishReasonStop,
+					},
+					{
+						Index: 1,
+						Message: openai.ChatCompletionResponseChoiceMessage{
+							Content: ptr.To("response part 2"),
+							Role:    "assistant",
+						},
+						FinishReason: openai.ChatCompletionChoicesFinishReasonStop,
+					},
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			body, err := json.Marshal(tt.bedrockResp)
+			require.NoError(t, err)
+
+			hm, bm, usedToken, err := o.ResponseBody(nil, bytes.NewBuffer(body), false)
+			require.NoError(t, err)
+			require.NotNil(t, bm)
+			require.NotNil(t, bm.Mutation)
+			require.NotNil(t, bm.Mutation.(*extprocv3.BodyMutation_Body))
+			newBody := bm.Mutation.(*extprocv3.BodyMutation_Body).Body
+			require.NotNil(t, newBody)
+			require.NotNil(t, hm)
+			require.NotNil(t, hm.SetHeaders)
+			require.Len(t, hm.SetHeaders, 1)
+			require.Equal(t, "content-length", hm.SetHeaders[0].Header.Key)
+			require.Equal(t, strconv.Itoa(len(newBody)), string(hm.SetHeaders[0].Header.RawValue))
+
+			var openAIResp openai.ChatCompletionResponse
+			err = json.Unmarshal(newBody, &openAIResp)
+			require.NoError(t, err)
+			require.Equal(t,
+				LLMTokenUsage{
+					InputTokens:  uint32(tt.expectedOutput.Usage.PromptTokens),     //nolint:gosec
+					OutputTokens: uint32(tt.expectedOutput.Usage.CompletionTokens), //nolint:gosec
+					TotalTokens:  uint32(tt.expectedOutput.Usage.TotalTokens),      //nolint:gosec
+				}, usedToken)
+			if !cmp.Equal(openAIResp, tt.expectedOutput) {
+				t.Errorf("ResponseBody(), diff(got, expected) = %s\n", cmp.Diff(openAIResp, tt.expectedOutput))
+			}
+		})
+	}
+}

tests/crdcel/testdata/backendsecuritypolicies/aws_credential_file.yaml

@@ -11,7 +11,8 @@ metadata:
 spec:
   type: AWSCredentials
   awsCredentials:
-    region: us-east-1
+#    region: us-east-1
+    region: eu-central-1
     credentialsFile:
       secretRef:
         name: placeholder

tests/crdcel/testdata/backendsecuritypolicies/aws_oidc.yaml

@@ -11,7 +11,8 @@ metadata:
 spec:
   type: AWSCredentials
   awsCredentials:
-    region: us-east-1
+#    region: us-east-1
+    region: eu-central-1
     oidcExchangeToken:
       awsRoleArn: placeholder
       oidc:

tests/crdcel/testdata/backendsecuritypolicies/multiple_security_policies.yaml

@@ -14,4 +14,6 @@ spec:
     secretRef:
       name: placeholder
   awsCredentials:
-    region: us-east-1
+#    region: us-east-1
+    region: eu-central-1
+