From 10a8187d9d89c4d60167d9c66206795ce7185b55 Mon Sep 17 00:00:00 2001 From: Francesco Pantano Date: Mon, 9 Sep 2024 13:02:56 +0200 Subject: [PATCH] Run glance-api as privileged container when image cache is enabled ImageCache currently uses two different cronJobs associated to each glance-api instance (-cleaner and -pruner cronJobs). They mount /var/lib/glance/image-cache, a RWO Pvc, and execute a glance utility on the filestem owned by glance kolla user/group. Without glance-api being privileged, after the cronJob execution the Glance Pod is not able to access the image-cache path anymore, resulting in a Permission denied error. This patch defines a FSGroup that should be set at Pod level to make sure we always have the right privileges on the container fs, and, in addition, it runs glance-api as a privileged container when Cache is enabled. Jira: https://issues.redhat.com/browse/OSPRH-9842 Signed-off-by: Francesco Pantano --- api/v1beta1/common_types.go | 1 + api/v1beta1/glance_types.go | 2 +- api/v1beta1/glance_webhook.go | 1 + controllers/glanceapi_controller.go | 2 ++ pkg/glance/const.go | 2 +- pkg/glance/funcs.go | 36 +++++++++++------------------ pkg/glanceapi/cachejob.go | 17 +++++++------- pkg/glanceapi/statefulset.go | 3 +++ 8 files changed, 31 insertions(+), 33 deletions(-) diff --git a/api/v1beta1/common_types.go b/api/v1beta1/common_types.go index c8106ffc..424090a5 100644 --- a/api/v1beta1/common_types.go +++ b/api/v1beta1/common_types.go @@ -108,6 +108,7 @@ type GlanceAPITemplate struct { APITimeout int `json:"apiTimeout,omitempty"` } +// Storage - type Storage struct { // +kubebuilder:validation:Optional // StorageClass - diff --git a/api/v1beta1/glance_types.go b/api/v1beta1/glance_types.go index ea661777..c9e8b1c7 100644 --- a/api/v1beta1/glance_types.go +++ b/api/v1beta1/glance_types.go @@ -35,7 +35,7 @@ const ( APIEdge = "edge" ) -// GlanceSpec defines the desired state of Glance +// GlanceSpecCore defines the desired state of Glance type GlanceSpecCore struct { // +kubebuilder:validation:Optional // +kubebuilder:default=glance diff --git a/api/v1beta1/glance_webhook.go b/api/v1beta1/glance_webhook.go index 754dac79..48bda50f 100644 --- a/api/v1beta1/glance_webhook.go +++ b/api/v1beta1/glance_webhook.go @@ -346,6 +346,7 @@ func (r *GlanceSpec) ValidateUpdate(old GlanceSpec, basePath *field.Path) field. return r.GlanceSpecCore.ValidateUpdate(old.GlanceSpecCore, basePath) } +// ValidateUpdate - func (r *GlanceSpecCore) ValidateUpdate(old GlanceSpecCore, basePath *field.Path) field.ErrorList { var allErrs field.ErrorList diff --git a/controllers/glanceapi_controller.go b/controllers/glanceapi_controller.go index 2ddd41e2..e6b72a71 100644 --- a/controllers/glanceapi_controller.go +++ b/controllers/glanceapi_controller.go @@ -802,6 +802,8 @@ func (r *GlanceAPIReconciler) reconcileNormal( // This is currently required because cleaner and pruner cronJobs // mount the same pvc to clean data present in /var/lib/glance/image-cache + // TODO (fpantano) reference a Glance spec/proposal to move to a different + // approach if len(instance.Spec.ImageCache.Size) > 0 { privileged = true } diff --git a/pkg/glance/const.go b/pkg/glance/const.go index 2022b648..9df824fc 100644 --- a/pkg/glance/const.go +++ b/pkg/glance/const.go @@ -54,7 +54,7 @@ const ( GlanceInternalPort int32 = 9292 // GlanceUID - https://github.com/openstack/kolla/blob/master/kolla/common/users.py GlanceUID int64 = 42415 - // GlanceGid - https://github.com/openstack/kolla/blob/master/kolla/common/users.py + // GlanceGID - https://github.com/openstack/kolla/blob/master/kolla/common/users.py GlanceGID int64 = 42415 // DefaultsConfigFileName - DefaultsConfigFileName = "00-config.conf" diff --git a/pkg/glance/funcs.go b/pkg/glance/funcs.go index 78767150..1955241f 100644 --- a/pkg/glance/funcs.go +++ b/pkg/glance/funcs.go @@ -2,6 +2,7 @@ package glance import ( corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -13,19 +14,16 @@ func GetOwningGlanceName(instance client.Object) string { return ownerRef.Name } } - return "" } // dbSyncSecurityContext - currently used to make sure we don't run db-sync as // root user func dbSyncSecurityContext() *corev1.SecurityContext { - runAsUser := int64(GlanceUID) - runAsGroup := int64(GlanceGID) return &corev1.SecurityContext{ - RunAsUser: &runAsUser, - RunAsGroup: &runAsGroup, + RunAsUser: ptr.To(GlanceUID), + RunAsGroup: ptr.To(GlanceGID), Capabilities: &corev1.Capabilities{ Drop: []corev1.Capability{ "MKNOD", @@ -40,16 +38,12 @@ func dbSyncSecurityContext() *corev1.SecurityContext { // BaseSecurityContext - currently used to make sure we don't run cronJob and Log // Pods as root user, and we drop privileges and Capabilities we don't need func BaseSecurityContext() *corev1.SecurityContext { - falseVal := false - trueVal := true - runAsUser := int64(GlanceUID) - runAsGroup := int64(GlanceGID) return &corev1.SecurityContext{ - RunAsUser: &runAsUser, - RunAsGroup: &runAsGroup, - RunAsNonRoot: &trueVal, - AllowPrivilegeEscalation: &falseVal, + RunAsUser: ptr.To(GlanceUID), + RunAsGroup: ptr.To(GlanceGID), + RunAsNonRoot: ptr.To(true), + AllowPrivilegeEscalation: ptr.To(false), Capabilities: &corev1.Capabilities{ Drop: []corev1.Capability{ "ALL", @@ -63,11 +57,10 @@ func BaseSecurityContext() *corev1.SecurityContext { // APISecurityContext - func APISecurityContext(userID int64, privileged bool) *corev1.SecurityContext { - runAsUser := int64(userID) - trueVal := true + return &corev1.SecurityContext{ - AllowPrivilegeEscalation: &trueVal, - RunAsUser: &runAsUser, + AllowPrivilegeEscalation: ptr.To(true), + RunAsUser: ptr.To(userID), Privileged: &privileged, SeccompProfile: &corev1.SeccompProfile{ Type: corev1.SeccompProfileTypeRuntimeDefault, @@ -77,16 +70,15 @@ func APISecurityContext(userID int64, privileged bool) *corev1.SecurityContext { // HttpdSecurityContext - func HttpdSecurityContext() *corev1.SecurityContext { - runAsUser := int64(0) - falseVal := false + return &corev1.SecurityContext{ - AllowPrivilegeEscalation: &falseVal, Capabilities: &corev1.Capabilities{ Drop: []corev1.Capability{ - "ALL", + "MKNOD", }, }, - RunAsUser: &runAsUser, + RunAsUser: ptr.To(GlanceUID), + RunAsGroup: ptr.To(GlanceGID), SeccompProfile: &corev1.SeccompProfile{ Type: corev1.SeccompProfileTypeRuntimeDefault, }, diff --git a/pkg/glanceapi/cachejob.go b/pkg/glanceapi/cachejob.go index 2f421d83..fa08d440 100644 --- a/pkg/glanceapi/cachejob.go +++ b/pkg/glanceapi/cachejob.go @@ -16,14 +16,13 @@ limitations under the License. package glanceapi import ( + "fmt" glancev1 "github.com/openstack-k8s-operators/glance-operator/api/v1beta1" "github.com/openstack-k8s-operators/glance-operator/pkg/glance" - - "fmt" - batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" ) // ImageCacheJob - @@ -31,7 +30,6 @@ func ImageCacheJob( instance *glancev1.GlanceAPI, cronSpec glance.CronJobSpec, ) *batchv1.CronJob { - runAsUser := int64(0) var config0644AccessMode int32 = 0644 cronCommand := fmt.Sprintf( @@ -100,6 +98,9 @@ func ImageCacheJob( Completions: &completions, Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ + SecurityContext: &corev1.PodSecurityContext{ + FSGroup: ptr.To(glance.GlanceUID), + }, Affinity: GetGlanceAPIPodAffinity(instance), Containers: []corev1.Container{ { @@ -108,11 +109,9 @@ func ImageCacheJob( Command: []string{ "/bin/bash", }, - Args: args, - VolumeMounts: cronJobVolumeMounts, - SecurityContext: &corev1.SecurityContext{ - RunAsUser: &runAsUser, - }, + Args: args, + VolumeMounts: cronJobVolumeMounts, + SecurityContext: glance.BaseSecurityContext(), }, }, Volumes: cronJobVolume, diff --git a/pkg/glanceapi/statefulset.go b/pkg/glanceapi/statefulset.go index 63f00063..e050b3e2 100644 --- a/pkg/glanceapi/statefulset.go +++ b/pkg/glanceapi/statefulset.go @@ -179,6 +179,9 @@ func StatefulSet( Labels: labels, }, Spec: corev1.PodSpec{ + SecurityContext: &corev1.PodSecurityContext{ + FSGroup: &userID, + }, ServiceAccountName: instance.Spec.ServiceAccount, // When using Cinder we run as privileged, but also some // commands need to be run on the host using nsenter (eg: