From b326c490d543f3f9dff2cb0b086f51e27037fdab Mon Sep 17 00:00:00 2001 From: Vasily Oleynikov Date: Fri, 19 Jul 2024 16:03:00 +0300 Subject: [PATCH] [controller] Add LocalStorageClass LVM volume group unique check (#57) Signed-off-by: v.oleynikov --- images/webhooks/src/go.mod | 13 +++- images/webhooks/src/go.sum | 38 ++++++++---- images/webhooks/src/handlers/func.go | 51 ++++++++++++++++ images/webhooks/src/handlers/lscValidator.go | 63 +++++++++++++++++++- templates/webhooks/rbac-for-us.yaml | 8 +++ 5 files changed, 157 insertions(+), 16 deletions(-) diff --git a/images/webhooks/src/go.mod b/images/webhooks/src/go.mod index c821aefb..618a8f6e 100644 --- a/images/webhooks/src/go.mod +++ b/images/webhooks/src/go.mod @@ -4,17 +4,21 @@ go 1.22.2 require ( github.com/deckhouse/sds-local-volume/api v0.0.0-20240704080736-a027a963cc2c + github.com/deckhouse/sds-node-configurator/api v0.0.0-20240718134550-8296fe5656e3 github.com/sirupsen/logrus v1.9.3 github.com/slok/kubewebhook/v2 v2.6.0 - k8s.io/api v0.30.0 - k8s.io/apimachinery v0.30.2 - k8s.io/client-go v0.30.0 + k8s.io/api v0.30.3 + k8s.io/apiextensions-apiserver v0.30.3 + k8s.io/apimachinery v0.30.3 + k8s.io/client-go v0.30.3 k8s.io/klog/v2 v2.120.1 + sigs.k8s.io/controller-runtime v0.18.4 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.12.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect @@ -24,12 +28,15 @@ require ( github.com/google/gnostic-models v0.6.8 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect golang.org/x/net v0.24.0 // indirect golang.org/x/oauth2 v0.19.0 // indirect golang.org/x/sys v0.19.0 // indirect diff --git a/images/webhooks/src/go.sum b/images/webhooks/src/go.sum index 096ba14a..b5d406d0 100644 --- a/images/webhooks/src/go.sum +++ b/images/webhooks/src/go.sum @@ -3,12 +3,18 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckhouse/sds-local-volume/api v0.0.0-20240704080736-a027a963cc2c h1:1hZBDe3aKqok/jOnbKfa5CMQ6viL/dsnsHujEaq+oxU= github.com/deckhouse/sds-local-volume/api v0.0.0-20240704080736-a027a963cc2c/go.mod h1:cYxHYJmIl6g9lXb1etqmLeQL/vsPMgscmact/FObd+U= +github.com/deckhouse/sds-node-configurator/api v0.0.0-20240718134550-8296fe5656e3 h1:zlGPvlNnEDCNlezto3a7ScVJzqu+fvrnGRpAsQs2wNM= +github.com/deckhouse/sds-node-configurator/api v0.0.0-20240718134550-8296fe5656e3/go.mod h1:H71+9G0Jr46Qs0BA3z3/xt0h9lbnJnCEYcaCJCWFBf0= github.com/emicklei/go-restful/v3 v3.12.0 h1:y2DdzBAURM29NFF94q6RaY4vjIH1rtwDapwQtU84iWk= github.com/emicklei/go-restful/v3 v3.12.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= +github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= @@ -33,6 +39,8 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJY github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -52,10 +60,10 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY= -github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= -github.com/onsi/gomega v1.31.0 h1:54UJxxj6cPInHS3a35wm6BK/F9nHYueZ1NVujHDrnXE= -github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk= +github.com/onsi/ginkgo/v2 v2.17.1 h1:V++EzdbhI4ZV4ev0UTIj0PzhzOcReJFyJaLjtSF55M8= +github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= +github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= +github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -77,9 +85,15 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -132,18 +146,22 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.30.0 h1:siWhRq7cNjy2iHssOB9SCGNCl2spiF1dO3dABqZ8niA= -k8s.io/api v0.30.0/go.mod h1:OPlaYhoHs8EQ1ql0R/TsUgaRPhpKNxIMrKQfWUp8QSE= -k8s.io/apimachinery v0.30.2 h1:fEMcnBj6qkzzPGSVsAZtQThU62SmQ4ZymlXRC5yFSCg= -k8s.io/apimachinery v0.30.2/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= -k8s.io/client-go v0.30.0 h1:sB1AGGlhY/o7KCyCEQ0bPWzYDL0pwOZO4vAtTSh/gJQ= -k8s.io/client-go v0.30.0/go.mod h1:g7li5O5256qe6TYdAMyX/otJqMhIiGgTapdLchhmOaY= +k8s.io/api v0.30.3 h1:ImHwK9DCsPA9uoU3rVh4QHAHHK5dTSv1nxJUapx8hoQ= +k8s.io/api v0.30.3/go.mod h1:GPc8jlzoe5JG3pb0KJCSLX5oAFIW3/qNJITlDj8BH04= +k8s.io/apiextensions-apiserver v0.30.3 h1:oChu5li2vsZHx2IvnGP3ah8Nj3KyqG3kRSaKmijhB9U= +k8s.io/apiextensions-apiserver v0.30.3/go.mod h1:uhXxYDkMAvl6CJw4lrDN4CPbONkF3+XL9cacCT44kV4= +k8s.io/apimachinery v0.30.3 h1:q1laaWCmrszyQuSQCfNB8cFgCuDAoPszKY4ucAjDwHc= +k8s.io/apimachinery v0.30.3/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k= +k8s.io/client-go v0.30.3/go.mod h1:8d4pf8vYu665/kUbsxWAQ/JDBNWqfFeZnvFiVdmx89U= k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240423202451-8948a665c108 h1:Q8Z7VlGhcJgBHJHYugJ/K/7iB8a2eSxCyxdVjJp+lLY= k8s.io/kube-openapi v0.0.0-20240423202451-8948a665c108/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/utils v0.0.0-20240423183400-0849a56e8f22 h1:ao5hUqGhsqdm+bYbjH/pRkCs0unBGe9UyDahzs9zQzQ= k8s.io/utils v0.0.0-20240423183400-0849a56e8f22/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw= +sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= diff --git a/images/webhooks/src/handlers/func.go b/images/webhooks/src/handlers/func.go index f0c92469..a1669960 100644 --- a/images/webhooks/src/handlers/func.go +++ b/images/webhooks/src/handlers/func.go @@ -18,7 +18,18 @@ package handlers import ( "context" + snc "github.com/deckhouse/sds-node-configurator/api/v1alpha1" + v1 "k8s.io/api/core/v1" + "k8s.io/api/resource/v1alpha2" + sv1 "k8s.io/api/storage/v1" + extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiruntime "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" "net/http" + "os" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/slok/kubewebhook/v2/pkg/log" @@ -29,6 +40,46 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +func NewKubeClient(kubeconfigPath string) (client.Client, error) { + var config *rest.Config + var err error + + if kubeconfigPath == "" { + kubeconfigPath = os.Getenv("kubeconfig") + } + + config, err = clientcmd.BuildConfigFromFlags("", kubeconfigPath) + + if err != nil { + return nil, err + } + + var ( + resourcesSchemeFuncs = []func(*apiruntime.Scheme) error{ + v1alpha2.AddToScheme, + snc.AddToScheme, + clientgoscheme.AddToScheme, + extv1.AddToScheme, + v1.AddToScheme, + sv1.AddToScheme, + } + ) + + scheme := apiruntime.NewScheme() + for _, f := range resourcesSchemeFuncs { + err = f(scheme) + if err != nil { + return nil, err + } + } + + clientOpts := client.Options{ + Scheme: scheme, + } + + return client.New(config, clientOpts) +} + func GetMutatingWebhookHandler(mutationFunc func(ctx context.Context, _ *model.AdmissionReview, obj metav1.Object) (*kwhmutating.MutatorResult, error), mutatorID string, obj metav1.Object, logger log.Logger) (http.Handler, error) { mutatorFunc := kwhmutating.MutatorFunc(mutationFunc) diff --git a/images/webhooks/src/handlers/lscValidator.go b/images/webhooks/src/handlers/lscValidator.go index 132a7bd2..256acdc2 100644 --- a/images/webhooks/src/handlers/lscValidator.go +++ b/images/webhooks/src/handlers/lscValidator.go @@ -18,7 +18,11 @@ package handlers import ( "context" + "fmt" slv "github.com/deckhouse/sds-local-volume/api/v1alpha1" + snc "github.com/deckhouse/sds-node-configurator/api/v1alpha1" + "k8s.io/klog/v2" + "slices" "github.com/slok/kubewebhook/v2/pkg/model" kwhvalidating "github.com/slok/kubewebhook/v2/pkg/webhook/validating" @@ -32,9 +36,56 @@ func LSCValidate(ctx context.Context, _ *model.AdmissionReview, obj metav1.Objec return &kwhvalidating.ValidatorResult{}, nil } + cl, err := NewKubeClient("") + if err != nil { + klog.Fatal(err) + } + + listDevice := &snc.LvmVolumeGroupList{ + TypeMeta: metav1.TypeMeta{ + Kind: snc.BlockDeviceKind, + APIVersion: snc.TypeMediaAPIVersion, + }, + ListMeta: metav1.ListMeta{}, + Items: []snc.LvmVolumeGroup{}, + } + + err = cl.List(ctx, listDevice) + if err != nil { + klog.Fatal(err) + } + + errMsg := "" + var lvmVolumeGroupUnique []string + thickExists := false thinExists := false for _, lvmGroup := range lsc.Spec.LVM.LVMVolumeGroups { + lvgExists := false + + if slices.Contains(lvmVolumeGroupUnique, lvmGroup.Name) { + errMsg = fmt.Sprintf("There must be unique LVMVolumeGroup names (%s duplicates)", lvmGroup.Name) + klog.Info(errMsg) + return &kwhvalidating.ValidatorResult{Valid: false, Message: errMsg}, + nil + } + + lvmVolumeGroupUnique = append(lvmVolumeGroupUnique, lvmGroup.Name) + + for _, lvmVG := range listDevice.Items { + if lvmVG.Name == lvmGroup.Name { + lvgExists = true + break + } + } + + if !lvgExists { + errMsg = fmt.Sprintf("LVMVolumeGroup %s not found; ", lvmGroup.Name) + klog.Info(errMsg) + return &kwhvalidating.ValidatorResult{Valid: false, Message: errMsg}, + nil + } + if lvmGroup.Thin == nil { thickExists = true } else { @@ -43,17 +94,23 @@ func LSCValidate(ctx context.Context, _ *model.AdmissionReview, obj metav1.Objec } if thinExists && lsc.Spec.LVM.Type == "Thick" { - return &kwhvalidating.ValidatorResult{Valid: false, Message: "there must be only thick pools with Thick LVM type"}, + errMsg = "There must be only thick pools with Thick LVM type" + klog.Info(errMsg) + return &kwhvalidating.ValidatorResult{Valid: false, Message: errMsg}, nil } if thickExists && lsc.Spec.LVM.Type == "Thin" { - return &kwhvalidating.ValidatorResult{Valid: false, Message: "there must be only thin pools with Thin LVM type"}, + errMsg = "There must be only thin pools with Thin LVM type" + klog.Info(errMsg) + return &kwhvalidating.ValidatorResult{Valid: false, Message: errMsg}, nil } if thickExists == true && thinExists == true { - return &kwhvalidating.ValidatorResult{Valid: false, Message: "there must be only thin or thick pools simultaneously"}, + errMsg = "There must be only thin or thick pools simultaneously" + klog.Info(errMsg) + return &kwhvalidating.ValidatorResult{Valid: false, Message: errMsg}, nil } else { return &kwhvalidating.ValidatorResult{Valid: true}, diff --git a/templates/webhooks/rbac-for-us.yaml b/templates/webhooks/rbac-for-us.yaml index 375e48a4..4cbf1cc0 100644 --- a/templates/webhooks/rbac-for-us.yaml +++ b/templates/webhooks/rbac-for-us.yaml @@ -12,6 +12,14 @@ metadata: name: d8:{{ .Chart.Name }}:webhooks {{- include "helm_lib_module_labels" (list . (dict "app" "webhooks")) | nindent 2 }} rules: + - verbs: + - get + - list + - watch + apiGroups: + - storage.deckhouse.io + resources: + - lvmvolumegroups - apiGroups: - "" verbs: