From 46977d15c8a9a30fc3b77b751ad32d1005bc53ba Mon Sep 17 00:00:00 2001 From: Giacomo Olivero Date: Wed, 13 Dec 2023 15:42:09 +0100 Subject: [PATCH] Tenant autoenrolling feature Tenant autoenrolling feature kc fix lint fix broken link fix --- .../certificate-provisioning/README.md | 2 +- operators/api/v1alpha1/workspace_types.go | 19 ++++++ operators/api/v1alpha2/common.go | 3 + operators/api/v1alpha2/tenant_types.go | 5 +- .../crds/crownlabs.polito.it_tenants.yaml | 1 + .../crds/crownlabs.polito.it_workspaces.yaml | 8 +++ .../tenant-controller/tenant_controller.go | 24 +++++-- .../tenant-controller/workspace_controller.go | 61 ++++++++++++++++++ operators/pkg/tenantwh/validating.go | 63 ++++++++++++++++++- operators/pkg/utils/common.go | 8 +++ 10 files changed, 184 insertions(+), 10 deletions(-) diff --git a/infrastructure/certificate-provisioning/README.md b/infrastructure/certificate-provisioning/README.md index 3ea80d131..fd52e4c03 100644 --- a/infrastructure/certificate-provisioning/README.md +++ b/infrastructure/certificate-provisioning/README.md @@ -134,7 +134,7 @@ helm install kubed appscode/kubed \ ### Secret synchronization -Once kubed is installed, secrets can be duplicated in multiple namespaces, and kept synchronized, by adding the ad-hoc annotation [[6]](https://appscode.com/products/kubed/v0.12.0-rc.2/welcome/): +Once kubed is installed, secrets can be duplicated in multiple namespaces, and kept synchronized, by adding the ad-hoc annotation [[6]](https://cert-manager.io/v1.1-docs/faq/kubed/#syncing-arbitrary-secrets-across-namespaces-using-kubed): ```yaml ... diff --git a/operators/api/v1alpha1/workspace_types.go b/operators/api/v1alpha1/workspace_types.go index d376bd1cd..1dc7ba809 100644 --- a/operators/api/v1alpha1/workspace_types.go +++ b/operators/api/v1alpha1/workspace_types.go @@ -24,11 +24,30 @@ import ( // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. +// +kubebuilder:validation:Enum="";withApproval;immediate + +// WorkspaceAutoenroll defines auto-enroll capabilities of the Workspace. +type WorkspaceAutoenroll string + +const ( + // AutoenrollNone -> no auto-enroll capabilities. + AutoenrollNone WorkspaceAutoenroll = "" + + // AutoenrollWithApproval -> auto-enroll is enabled, but requires approval by a manager. + AutoenrollWithApproval WorkspaceAutoenroll = "withApproval" + + // AutoenrollImmediate -> auto-enroll is enabled, and the user can enroll himself/herself. + AutoenrollImmediate WorkspaceAutoenroll = "immediate" +) + // WorkspaceSpec is the specification of the desired state of the Workspace. type WorkspaceSpec struct { // The human-readable name of the Workspace. PrettyName string `json:"prettyName"` + // AutoEnroll capability definition. If omitted, no autoenroll features will be added. + AutoEnroll WorkspaceAutoenroll `json:"autoEnroll,omitempty"` + // The amount of resources associated with this workspace, and inherited by enrolled tenants. Quota WorkspaceResourceQuota `json:"quota"` } diff --git a/operators/api/v1alpha2/common.go b/operators/api/v1alpha2/common.go index 247e7d29f..716342d04 100644 --- a/operators/api/v1alpha2/common.go +++ b/operators/api/v1alpha2/common.go @@ -52,5 +52,8 @@ const ( // WorkspaceLabelPrefix is the prefix of a label assigned to a tenant indicating it is subscribed to a workspace. const WorkspaceLabelPrefix = "crownlabs.polito.it/workspace-" +// WorkspaceLabelAutoenroll is the label assigned to a workspace in which autoenroll is enabled. +const WorkspaceLabelAutoenroll = "crownlabs.polito.it/autoenroll" + // TnOperatorFinalizerName is the name of the finalizer corresponding to the tenant operator. const TnOperatorFinalizerName = "crownlabs.polito.it/tenant-operator" diff --git a/operators/api/v1alpha2/tenant_types.go b/operators/api/v1alpha2/tenant_types.go index 69eb7aed6..f801591df 100644 --- a/operators/api/v1alpha2/tenant_types.go +++ b/operators/api/v1alpha2/tenant_types.go @@ -23,7 +23,7 @@ import ( // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. -// +kubebuilder:validation:Enum=manager;user +// +kubebuilder:validation:Enum=manager;user;candidate // WorkspaceUserRole is an enumeration of the different roles that can be // associated to a Tenant in a Workspace. @@ -36,6 +36,9 @@ const ( // User -> a Tenant with User role can only interact with his/her own // environments (e.g. VMs) within that Workspace. User WorkspaceUserRole = "user" + // Candidate -> a Tenant with Candidate role wants to be added to the + // Workspace, but he/she is not yet enrolled. + Candidate WorkspaceUserRole = "candidate" // SVCTenantName -> name of a system/service tenant to which other resources might belong. SVCTenantName string = "service-tenant" diff --git a/operators/deploy/crds/crownlabs.polito.it_tenants.yaml b/operators/deploy/crds/crownlabs.polito.it_tenants.yaml index de860c7ed..289542e50 100644 --- a/operators/deploy/crds/crownlabs.polito.it_tenants.yaml +++ b/operators/deploy/crds/crownlabs.polito.it_tenants.yaml @@ -132,6 +132,7 @@ spec: enum: - manager - user + - candidate type: string required: - name diff --git a/operators/deploy/crds/crownlabs.polito.it_workspaces.yaml b/operators/deploy/crds/crownlabs.polito.it_workspaces.yaml index 63e63cda8..284a87955 100644 --- a/operators/deploy/crds/crownlabs.polito.it_workspaces.yaml +++ b/operators/deploy/crds/crownlabs.polito.it_workspaces.yaml @@ -49,6 +49,14 @@ spec: description: WorkspaceSpec is the specification of the desired state of the Workspace. properties: + autoEnroll: + description: AutoEnroll capability definition. If omitted, no autoenroll + features will be added. + enum: + - "" + - withApproval + - immediate + type: string prettyName: description: The human-readable name of the Workspace. type: string diff --git a/operators/pkg/tenant-controller/tenant_controller.go b/operators/pkg/tenant-controller/tenant_controller.go index b1b887083..1f7a713fb 100644 --- a/operators/pkg/tenant-controller/tenant_controller.go +++ b/operators/pkg/tenant-controller/tenant_controller.go @@ -153,7 +153,7 @@ func (r *TenantReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr tn.Status.Subscriptions = make(map[string]crownlabsv1alpha2.SubscriptionStatus, 1) } - tenantExistingWorkspaces, workspaces, err := r.checkValidWorkspaces(ctx, &tn) + tenantExistingWorkspaces, workspaces, enrolledWorkspaces, err := r.checkValidWorkspaces(ctx, &tn) if err != nil { retrigErr = err @@ -181,7 +181,7 @@ func (r *TenantReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr return ctrl.Result{}, err } - if err = r.handleKeycloakSubscription(ctx, &tn, tenantExistingWorkspaces); err != nil { + if err = r.handleKeycloakSubscription(ctx, &tn, enrolledWorkspaces); err != nil { klog.Errorf("Error when updating keycloak subscription for tenant %s -> %s", tn.Name, err) tn.Status.Subscriptions["keycloak"] = crownlabsv1alpha2.SubscrFailed retrigErr = err @@ -275,8 +275,11 @@ func (r *TenantReconciler) handleDeletion(ctx context.Context, tnName string) er } // checkValidWorkspaces check validity of workspaces in tenant. -func (r *TenantReconciler) checkValidWorkspaces(ctx context.Context, tn *crownlabsv1alpha2.Tenant) ([]crownlabsv1alpha2.TenantWorkspaceEntry, []crownlabsv1alpha1.Workspace, error) { +// allWsEntry []TenantWorkspaceEntry and allWs []Workspace contains all the workspaces associated with the tenant. +// enrolledWs []TenantWorkspaceEntry contains only the workspaces the tenant is enrolled in (`user` or `manager`). +func (r *TenantReconciler) checkValidWorkspaces(ctx context.Context, tn *crownlabsv1alpha2.Tenant) (allWsEntry []crownlabsv1alpha2.TenantWorkspaceEntry, allWs []crownlabsv1alpha1.Workspace, enrolledWs []crownlabsv1alpha2.TenantWorkspaceEntry, retErr error) { tenantExistingWorkspaces := []crownlabsv1alpha2.TenantWorkspaceEntry{} + enrolledWorkspaces := []crownlabsv1alpha2.TenantWorkspaceEntry{} workspaces := []crownlabsv1alpha1.Workspace{} tn.Status.FailingWorkspaces = []string{} var err error @@ -284,17 +287,26 @@ func (r *TenantReconciler) checkValidWorkspaces(ctx context.Context, tn *crownla for _, tnWs := range tn.Spec.Workspaces { wsLookupKey := types.NamespacedName{Name: tnWs.Name} var ws crownlabsv1alpha1.Workspace - if err = r.Get(ctx, wsLookupKey, &ws); err != nil { + err = r.Get(ctx, wsLookupKey, &ws) + switch { + case err != nil: // if there was a problem, add the workspace to the status of the tenant klog.Errorf("Error when checking if workspace %s exists in tenant %s -> %s", tnWs.Name, tn.Name, err) tn.Status.FailingWorkspaces = append(tn.Status.FailingWorkspaces, tnWs.Name) tnOpinternalErrors.WithLabelValues("tenant", "workspace-not-exist").Inc() - } else { + case tnWs.Role == crownlabsv1alpha2.Candidate && ws.Spec.AutoEnroll != crownlabsv1alpha1.AutoenrollWithApproval: + // Candidate role is allowed only if the workspace has autoEnroll = WithApproval + klog.Errorf("Workspace %s has not autoEnroll with approval, Candidate role is not allowed in tenant %s", tnWs.Name, tn.Name) + tn.Status.FailingWorkspaces = append(tn.Status.FailingWorkspaces, tnWs.Name) + default: tenantExistingWorkspaces = append(tenantExistingWorkspaces, tnWs) workspaces = append(workspaces, ws) + if tnWs.Role != crownlabsv1alpha2.Candidate { + enrolledWorkspaces = append(enrolledWorkspaces, tnWs) + } } } - return tenantExistingWorkspaces, workspaces, err + return tenantExistingWorkspaces, workspaces, enrolledWorkspaces, err } // deleteClusterNamespace deletes the namespace for the tenant, if it fails then it returns an error. diff --git a/operators/pkg/tenant-controller/workspace_controller.go b/operators/pkg/tenant-controller/workspace_controller.go index 20be3b2ab..bb42bf8f1 100644 --- a/operators/pkg/tenant-controller/workspace_controller.go +++ b/operators/pkg/tenant-controller/workspace_controller.go @@ -135,6 +135,14 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( tnOpinternalErrors.WithLabelValues("workspace", "cluster-resources").Inc() } + // handling autoEnrollment + err = r.handleAutoEnrollment(ctx, &ws) + if err != nil { + klog.Errorf("Error when handling autoEnrollment for workspace %s -> %s", ws.Name, err) + retrigErr = err + tnOpinternalErrors.WithLabelValues("workspace", "auto-enrollment").Inc() + } + if ws.Status.Subscriptions == nil { // len 1 of the map is for the number of subscriptions (keycloak) ws.Status.Subscriptions = make(map[string]crownlabsv1alpha2.SubscriptionStatus, 1) @@ -234,6 +242,59 @@ func removeWsFromTn(workspaces *[]crownlabsv1alpha2.TenantWorkspaceEntry, wsToRe } } +func (r *WorkspaceReconciler) handleAutoEnrollment(ctx context.Context, ws *crownlabsv1alpha1.Workspace) error { + // check label and update if needed + var wantedLabel string + if utils.AutoEnrollEnabled(ws.Spec.AutoEnroll) { + wantedLabel = string(ws.Spec.AutoEnroll) + } else { + wantedLabel = "disabled" + } + + if ws.Labels[crownlabsv1alpha2.WorkspaceLabelAutoenroll] != wantedLabel { + ws.Labels[crownlabsv1alpha2.WorkspaceLabelAutoenroll] = wantedLabel + + if err := r.Update(ctx, ws); err != nil { + klog.Errorf("Error when updating workspace %s -> %s", ws.Name, err) + return err + } + } + + // if actual AutoEnroll is WithApproval, nothing left to do + if ws.Spec.AutoEnroll == crownlabsv1alpha1.AutoenrollWithApproval { + return nil + } + + // if actual AutoEnroll is not WithApproval, manage Tenants in candidate status + var tenantsToUpdate crownlabsv1alpha2.TenantList + targetLabel := fmt.Sprintf("%s%s", crownlabsv1alpha2.WorkspaceLabelPrefix, ws.Name) + err := r.List(ctx, &tenantsToUpdate, &client.MatchingLabels{targetLabel: string(crownlabsv1alpha2.Candidate)}) + if err != nil { + klog.Errorf("Error when listing tenants subscribed to workspace %s -> %s", ws.Name, err) + return err + } + for i := range tenantsToUpdate.Items { + patch := client.MergeFrom(tenantsToUpdate.Items[i].DeepCopy()) + removeWsFromTn(&tenantsToUpdate.Items[i].Spec.Workspaces, ws.Name) + // if AutoEnrollment is Immediate, add the workspace with User role + if ws.Spec.AutoEnroll == crownlabsv1alpha1.AutoenrollImmediate { + tenantsToUpdate.Items[i].Spec.Workspaces = append( + tenantsToUpdate.Items[i].Spec.Workspaces, + crownlabsv1alpha2.TenantWorkspaceEntry{ + Name: ws.Name, + Role: crownlabsv1alpha2.User, + }, + ) + } + if err := r.Patch(ctx, &tenantsToUpdate.Items[i], patch); err != nil { + klog.Errorf("Error when updating tenant %s -> %s", tenantsToUpdate.Items[i].Name, err) + return err + } + } + + return nil +} + func (r *WorkspaceReconciler) createOrUpdateClusterResources(ctx context.Context, ws *crownlabsv1alpha1.Workspace, nsName string) (nsOk bool, err error) { ns := v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: nsName}} diff --git a/operators/pkg/tenantwh/validating.go b/operators/pkg/tenantwh/validating.go index 7646b4e52..2984e6bbd 100644 --- a/operators/pkg/tenantwh/validating.go +++ b/operators/pkg/tenantwh/validating.go @@ -30,6 +30,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + clv1alpha1 "github.com/netgroup-polito/CrownLabs/operators/api/v1alpha1" clv1alpha2 "github.com/netgroup-polito/CrownLabs/operators/api/v1alpha2" "github.com/netgroup-polito/CrownLabs/operators/pkg/utils" ) @@ -86,7 +87,10 @@ func (tv *TenantValidator) Handle(ctx context.Context, req admission.Request) ad return tv.HandleWorkspaceEdit(ctx, tenant, oldTenant, manager, req.Operation) } -// HandleSelfEdit checks every field but public keys for changes through DeepEqual. +// HandleSelfEdit checks every field but public keys for changes: +// - LastLogin must be within a certain tolerance; +// - Workspaces can be changed only if autoenroll is enabled and within the allowed roles; +// - Other fields must be unchanged. func (tv *TenantValidator) HandleSelfEdit(ctx context.Context, newTenant, oldTenant *clv1alpha2.Tenant) admission.Response { log := ctrl.LoggerFrom(ctx) newTenant.Spec.PublicKeys = nil @@ -99,15 +103,70 @@ func (tv *TenantValidator) HandleSelfEdit(ctx context.Context, newTenant, oldTen newTenant.Spec.LastLogin = metav1.Time{} oldTenant.Spec.LastLogin = metav1.Time{} + // manage workspaces + newWorkspaces := newTenant.Spec.Workspaces + oldWorkspaces := oldTenant.Spec.Workspaces + newTenant.Spec.Workspaces = nil + oldTenant.Spec.Workspaces = nil + if !reflect.DeepEqual(newTenant.Spec, oldTenant.Spec) { log.Info("denied: unexpected tenant spec change") - return admission.Denied("only changes to public keys are allowed in the owned tenant") + return admission.Denied("only changes to public keys or workspaces that have autoenroll enabled are allowed in the owned tenant") + } + + newTenant.Spec.Workspaces = newWorkspaces + oldTenant.Spec.Workspaces = oldWorkspaces + + res, err := tv.checkValidWorkspaces(ctx, newTenant, oldTenant) + if err != nil { + log.Error(err, "failed to check workspace changes") + return admission.Errored(http.StatusInternalServerError, err) + } + if !res { + log.Info("denied: workspaces validation failed") + return admission.Denied("you have changed workspaces you are not allowed to change") } log.Info("allowed") return admission.Allowed("") } +// checkValidWorkspaces checks that the user is not changing workspaces they are not allowed to change. +func (tv *TenantValidator) checkValidWorkspaces(ctx context.Context, newTenant, oldTenant *clv1alpha2.Tenant) (bool, error) { + workspaceDiff := CalculateWorkspacesDiff(newTenant, oldTenant) + newWorkspacesMap := mapFromWorkspacesList(newTenant) + + for ws, changed := range workspaceDiff { + if !changed { + // it's always ok to keep the same role + continue + } + wsObj := clv1alpha1.Workspace{} + err := tv.Client.Get(ctx, client.ObjectKey{Name: ws}, &wsObj) + if err != nil { + return false, fmt.Errorf("failed to fetch workspace %s: %w", ws, err) + } + if !utils.AutoEnrollEnabled(wsObj.Spec.AutoEnroll) { + // Tenant cannot change workspaces with autoenroll disabled + return false, nil + } + if _, ok := newWorkspacesMap[ws]; !ok { + // it's always possible to remove a Workspace from the Tenant if the target Workspace has autoenroll enabled + continue + } + if wsObj.Spec.AutoEnroll == clv1alpha1.AutoenrollImmediate && newWorkspacesMap[ws] != clv1alpha2.User { + // if AutoEnroll is Immediate, then the user has to enroll with User role + return false, nil + } + if wsObj.Spec.AutoEnroll == clv1alpha1.AutoenrollWithApproval && newWorkspacesMap[ws] != clv1alpha2.Candidate { + // if AutoEnroll is WithApproval, then the user has to enroll with Candidate role (to be approved by a Manager) + return false, nil + } + } + + return true, nil +} + // HandleWorkspaceEdit checks that changes made to the workspaces have been made by a valid manager, then checks other fields not to have been modified through DeepEqual. func (tv *TenantValidator) HandleWorkspaceEdit(ctx context.Context, newTenant, oldTenant, manager *clv1alpha2.Tenant, operation admissionv1.Operation) admission.Response { log := ctrl.LoggerFrom(ctx) diff --git a/operators/pkg/utils/common.go b/operators/pkg/utils/common.go index 4d4abe204..77bb79740 100644 --- a/operators/pkg/utils/common.go +++ b/operators/pkg/utils/common.go @@ -25,6 +25,8 @@ import ( "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + + clv1alpha1 "github.com/netgroup-polito/CrownLabs/operators/api/v1alpha1" ) // ParseDockerDirectory returns a valid Docker image directory. @@ -90,3 +92,9 @@ func CheckSingleLabel(obj client.Object, label, value string) bool { labels := obj.GetLabels() return labels != nil && labels[label] == value } + +// AutoEnrollEnabled checks if the specified WorkspaceAutoenroll enables any feature. +func AutoEnrollEnabled(autoEnroll clv1alpha1.WorkspaceAutoenroll) bool { + return autoEnroll == clv1alpha1.AutoenrollImmediate || + autoEnroll == clv1alpha1.AutoenrollWithApproval +}