feat: add service account impersonation support for controllers (#59)

Add support for per namespace service account impersonation in controllers.
This enables running controller operations with different service account
permissions based on namespace configuration.

Key changes:
- Add `ServiceAccounts` field to `ResourceGroupSpec` for mapping namespaces
  to service accounts
- Implement dynamic client creation with service account impersonation
- Add metrics for tracking impersonation `success`/`failures` and `latency`
- Support default service account fallback using `"*"` key

e.g usage:
```yaml
apiVersion: x.symphony.k8s.aws/v1alpha1
kind: ResourceGroup
metadata:
  name: deploymentservice.x.symphony.k8s.aws
spec:
  # NOTE: this is just an initial implementation, we'll probably
  # review and redesign this.
  serviceAccounts:
    production: prod-sa
    staging: staging-sa
    "*": fallback-sa
```

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

api/v1alpha1/resource_group.go

@@ -18,14 +18,19 @@ import (
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
+const (
+	// DefaultServiceAccountKey is the key to use for the default service account
+	// in the serviceAccounts map.
+	DefaultServiceAccountKey = "*"
+)
+
 // ResourceGroupSpec defines the desired state of ResourceGroup
 type ResourceGroupSpec struct {
 	// The kind of the resourcegroup. This is used to generate
 	// and create the CRD for the resourcegroup.
 	//
 	// +kubebuilder:validation:Required
 	Kind string `json:"kind,omitempty"`
-
 	// The APIVersion of the resourcegroup. This is used to generate
 	// and create the CRD for the resourcegroup.
 	//
@@ -40,6 +45,13 @@ type ResourceGroupSpec struct {
 	//
 	// +kubebuilder:validation:Optional
 	Resources []*Resource `json:"resources,omitempty"`
+	// ServiceAccount configuration for controller impersonation.
+	// Key is the namespace, value is the service account name to use.
+	// Special key "*" defines the default service account for any
+	// namespace not explicitly mapped.
+	//
+	// +kubebuilder:validation:Optional
+	ServiceAccounts map[string]string `json:"serviceAccounts,omitempty"`
 }
 
 // Definition represents the attributes that define an instance of

internal/controller/instance/controller.go

@@ -20,13 +20,16 @@ import (
 	"time"
 
 	"github.com/go-logr/logr"
+	"github.com/prometheus/client_golang/prometheus"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/dynamic"
 	ctrl "sigs.k8s.io/controller-runtime"
 
+	"github.com/aws-controllers-k8s/symphony/api/v1alpha1"
 	"github.com/aws-controllers-k8s/symphony/internal/k8smetadata"
+	"github.com/aws-controllers-k8s/symphony/internal/kubernetes"
 	"github.com/aws-controllers-k8s/symphony/internal/resourcegroup/graph"
 )
 
@@ -83,6 +86,8 @@ type Controller struct {
 	// reconcileConfig holds the configuration parameters for the reconciliation
 	// process.
 	reconcileConfig ReconcileConfig
+	// serviceAccounts is a map of service accounts to use for controller impersonation.
+	serviceAccounts map[string]string
 }
 
 // NewController creates a new Controller instance.
@@ -92,6 +97,7 @@ func NewController(
 	gvr schema.GroupVersionResource,
 	rg *graph.Graph,
 	client dynamic.Interface,
+	serviceAccounts map[string]string,
 	instanceLabeler k8smetadata.Labeler,
 ) *Controller {
 	return &Controller{
@@ -101,6 +107,7 @@ func NewController(
 		rg:              rg,
 		instanceLabeler: instanceLabeler,
 		reconcileConfig: reconcileConfig,
+		serviceAccounts: serviceAccounts,
 	}
 }
 
@@ -135,10 +142,17 @@ func (c *Controller) Reconcile(ctx context.Context, req ctrl.Request) error {
 		return fmt.Errorf("failed to create instance sub-resources labeler: %w", err)
 	}
 
+	// If possible, use a service account to create the execution client
+	// TODO(a-hilaly): client caching
+	executionClient, err := c.getExecutionClient(namespace)
+	if err != nil {
+		return fmt.Errorf("failed to create execution client: %w", err)
+	}
+
 	instanceGraphReconciler := &instanceGraphReconciler{
 		log:                         log,
 		gvr:                         c.gvr,
-		client:                      c.client,
+		client:                      executionClient,
 		runtime:                     rgRuntime,
 		instanceLabeler:             c.instanceLabeler,
 		instanceSubResourcesLabeler: instanceSubResourcesLabeler,
@@ -156,3 +170,98 @@ func getNamespaceName(req ctrl.Request) (string, string) {
 	}
 	return namespace, name
 }
+
+// errorCategory helps classify different types of impersonation errors
+type errorCategory string
+
+const (
+	errorConfigCreate errorCategory = "config_create"
+	errorInvalidSA    errorCategory = "invalid_sa"
+	errorClientCreate errorCategory = "client_create"
+	errorPermissions  errorCategory = "permissions"
+)
+
+// getExecutionClient determines the execution client to use for the instance.
+// If the instance is created in a namespace of which a service account is specified,
+// the execution client will be created using the service account. If no service account
+// is specified for the namespace, the default client will be used.
+func (c *Controller) getExecutionClient(namespace string) (dynamic.Interface, error) {
+	// if no service accounts are specified, use the default client
+	if len(c.serviceAccounts) == 0 {
+		c.log.V(1).Info("no service accounts configured, using default client")
+		return c.client, nil
+	}
+
+	timer := prometheus.NewTimer(impersonationDuration.WithLabelValues(namespace, ""))
+	defer timer.ObserveDuration()
+
+	// Check for namespace specific service account
+	if sa, ok := c.serviceAccounts[namespace]; ok {
+		userName, err := getServiceAccountUserName(namespace, sa)
+		if err != nil {
+			c.handleImpersonateError(namespace, sa, err)
+			return nil, fmt.Errorf("invalid service account configuration: %w", err)
+		}
+
+		client, err := kubernetes.NewDynamicClient(userName)
+		if err != nil {
+			c.handleImpersonateError(namespace, sa, err)
+			return nil, fmt.Errorf("failed to create impersonated client: %w", err)
+		}
+
+		impersonationTotal.WithLabelValues(namespace, sa, "success").Inc()
+		return client, nil
+	}
+
+	// Check for default service account (marked by "*")
+	if defaultSA, ok := c.serviceAccounts[v1alpha1.DefaultServiceAccountKey]; ok {
+		userName, err := getServiceAccountUserName(namespace, defaultSA)
+		if err != nil {
+			c.handleImpersonateError(namespace, defaultSA, err)
+			return nil, fmt.Errorf("invalid default service account configuration: %w", err)
+		}
+
+		client, err := kubernetes.NewDynamicClient(userName)
+		if err != nil {
+			c.handleImpersonateError(namespace, defaultSA, err)
+			return nil, fmt.Errorf("failed to create impersonated client with default SA: %w", err)
+		}
+
+		impersonationTotal.WithLabelValues(namespace, defaultSA, "success").Inc()
+		return client, nil
+	}
+
+	impersonationTotal.WithLabelValues(namespace, "", "default").Inc()
+	// Fallback to the default client
+	return c.client, nil
+}
+
+// handleImpersonateError logs the error and records the error in the metrics
+func (c *Controller) handleImpersonateError(namespace, sa string, err error) {
+	var category errorCategory
+	switch {
+	case strings.Contains(err.Error(), "forbidden"):
+		category = errorPermissions
+	case strings.Contains(err.Error(), "cannot get token"):
+		category = errorConfigCreate
+	default:
+		category = errorClientCreate
+	}
+	recordImpersonateError(namespace, sa, category)
+	c.log.Error(
+		err,
+		"failed to create impersonated client",
+		"namespace", namespace,
+		"serviceAccount", sa,
+		"errorCategory", category,
+	)
+}
+
+// getServiceAccountUserName builds the impersonate service account user name.
+// The format of the user name is "system:serviceaccount:<namespace>:<serviceaccount>"
+func getServiceAccountUserName(namespace, serviceAccount string) (string, error) {
+	if namespace == "" || serviceAccount == "" {
+		return "", fmt.Errorf("namespace and service account must be provided")
+	}
+	return fmt.Sprintf("system:serviceaccount:%s:%s", namespace, serviceAccount), nil
+}

internal/controller/instance/metrics.go

@@ -0,0 +1,69 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//     http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package instance
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+const (
+	// MetricImpersonationTotal is the total number of impersonation requests
+	// made by the controller
+	MetricImpersonationTotal = "controller_impersonation_total"
+	// MetricImpersonationErrors is the total number of errors encountered
+	// while making impersonation requests
+	MetricImpersonationErrors = "controller_impersonation_errors_total"
+	// MetricImpersonationDuration tracks the duration of impersonation operations
+	MetricImpersonationDuration = "controller_impersonation_duration_seconds"
+)
+
+var (
+	impersonationTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: MetricImpersonationTotal,
+			Help: "Total number of service account impersonation attempts by namespace and result",
+		},
+		[]string{"namespace", "service_account", "result"},
+	)
+
+	impersonationErrors = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: MetricImpersonationErrors,
+			Help: "Total number of service account impersonation errors by category",
+		},
+		[]string{"namespace", "service_account", "error_type"},
+	)
+
+	impersonationDuration = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    MetricImpersonationDuration,
+			Help:    "Duration of service account impersonation operations",
+			Buckets: []float64{0.01, 0.1, 0.5, 1, 2, 5},
+		},
+		[]string{"namespace", "service_account"},
+	)
+)
+
+func recordImpersonateError(namespace, sa string, category errorCategory) {
+	impersonationErrors.WithLabelValues(namespace, sa, string(category)).Inc()
+}
+
+func init() {
+	metrics.Registry.MustRegister(
+		impersonationTotal,
+		impersonationErrors,
+		impersonationDuration,
+	)
+}

internal/controller/resourcegroup/controller_reconcile.go

@@ -67,6 +67,7 @@ func (r *ResourceGroupReconciler) reconcileResourceGroup(ctx context.Context, rg
 		gvr,
 		processedRG,
 		r.dynamicClient,
+		rg.Spec.ServiceAccounts,
 		graphExecLabeler,
 	)
 

internal/kubernetes/clients.go

@@ -41,3 +41,25 @@ func NewClients() (*rest.Config, *kubernetes.Clientset, *dynamic.DynamicClient,
 	}
 	return config, clientset, dynamicClient, apiExtensionsClient, nil
 }
+
+// NewDynamicClient creates a new dynamic client with optional service account
+// impersonation
+func NewDynamicClient(impersonateUser string) (*dynamic.DynamicClient, error) {
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	if impersonateUser != "" {
+		config.Impersonate = rest.ImpersonationConfig{
+			UserName: impersonateUser,
+		}
+	}
+
+	dynamicClient, err := dynamic.NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
+
+	return dynamicClient, nil
+}

internal/kubernetes/clients_dev.go

@@ -58,3 +58,24 @@ func NewClients() (*rest.Config, *kubernetes.Clientset, dynamic.Interface, *apie
 	}
 	return config, clientset, dynamicClient, apiExtensionsClient, nil
 }
+
+// NewDynamicClient creates a new dynamic client with optional service account
+// impersonation
+func NewDynamicClient(impersonateUser string) (*dynamic.DynamicClient, error) {
+	config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
+	if err != nil {
+		return nil, err
+	}
+
+	if impersonateUser != "" {
+		config.Impersonate = rest.ImpersonationConfig{
+			UserName: impersonateUser,
+		}
+	}
+
+	dynamicClient, err := dynamic.NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
+	return dynamicClient, nil
+}