From 8752c91d04b18232edb1bbdfe9475402e17dda43 Mon Sep 17 00:00:00 2001 From: Viktor Kramarenko Date: Fri, 29 Mar 2024 19:58:45 +0300 Subject: [PATCH 01/21] first draft Signed-off-by: Viktor Kramarenko --- .../sds-local-volume-controller/cmd/main.go | 2 - .../Dockerfile | 1 + .../cmd/cmd/root.go | 74 ++++++- .../pkg/cache/cache.go | 151 +++++++++++++++ .../pkg/controller/lvg_watcher_cache.go | 182 ++++++++++++++++++ .../pkg/controller/pvc_watcher_cache.go | 174 +++++++++++++++++ .../pkg/scheduler/filter.go | 24 +-- .../pkg/scheduler/prioritize.go | 10 +- .../pkg/scheduler/route.go | 49 ++++- .../rbac-for-us.yaml | 5 +- 10 files changed, 639 insertions(+), 33 deletions(-) create mode 100644 images/sds-local-volume-scheduler-extender/pkg/cache/cache.go create mode 100644 images/sds-local-volume-scheduler-extender/pkg/controller/lvg_watcher_cache.go create mode 100644 images/sds-local-volume-scheduler-extender/pkg/controller/pvc_watcher_cache.go diff --git a/images/sds-local-volume-controller/cmd/main.go b/images/sds-local-volume-controller/cmd/main.go index 5fbc6b97..0204fc55 100644 --- a/images/sds-local-volume-controller/cmd/main.go +++ b/images/sds-local-volume-controller/cmd/main.go @@ -118,6 +118,4 @@ func main() { log.Error(err, "[main] unable to mgr.Start") os.Exit(1) } - - log.Info("[main] successfully starts the manager") } diff --git a/images/sds-local-volume-scheduler-extender/Dockerfile b/images/sds-local-volume-scheduler-extender/Dockerfile index 256a9c3f..cf5afcce 100644 --- a/images/sds-local-volume-scheduler-extender/Dockerfile +++ b/images/sds-local-volume-scheduler-extender/Dockerfile @@ -17,5 +17,6 @@ RUN GOOS=linux GOARCH=amd64 go build -o sds-local-volume-scheduler-extender FROM --platform=linux/amd64 $BASE_GOLANG_20_ALPINE COPY --from=builder /go/src/cmd/sds-local-volume-scheduler-extender /go/src/cmd/sds-local-volume-scheduler-extender +RUN apk add curl ENTRYPOINT ["/go/src/cmd/sds-local-volume-scheduler-extender"] diff --git a/images/sds-local-volume-scheduler-extender/cmd/cmd/root.go b/images/sds-local-volume-scheduler-extender/cmd/cmd/root.go index d7c9e1a2..384e596e 100644 --- a/images/sds-local-volume-scheduler-extender/cmd/cmd/root.go +++ b/images/sds-local-volume-scheduler-extender/cmd/cmd/root.go @@ -24,21 +24,23 @@ import ( "os" "os/signal" "sds-local-volume-scheduler-extender/api/v1alpha1" + "sds-local-volume-scheduler-extender/pkg/cache" + "sds-local-volume-scheduler-extender/pkg/controller" "sds-local-volume-scheduler-extender/pkg/kubutils" "sds-local-volume-scheduler-extender/pkg/logger" "sds-local-volume-scheduler-extender/pkg/scheduler" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/manager" "sync" "syscall" "time" + "github.com/spf13/cobra" v1 "k8s.io/api/core/v1" sv1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/spf13/cobra" apiruntime "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/yaml" ) @@ -53,18 +55,21 @@ var resourcesSchemeFuncs = []func(*apiruntime.Scheme) error{ const ( defaultDivisor = 1 defaultListenAddr = ":8000" + defaultCacheSize = 10 ) type Config struct { ListenAddr string `json:"listen"` DefaultDivisor float64 `json:"default-divisor"` LogLevel string `json:"log-level"` + CacheSize int `json:"cache-size"` } var config = &Config{ ListenAddr: defaultListenAddr, DefaultDivisor: defaultDivisor, LogLevel: "2", + CacheSize: defaultCacheSize, } var rootCmd = &cobra.Command{ @@ -122,22 +127,62 @@ func subMain(parentCtx context.Context) error { } log.Info("[subMain] successfully read scheme CR") - cl, err := client.New(kConfig, client.Options{ - Scheme: scheme, - WarningHandler: client.WarningHandlerOptions{}, - }) + managerOpts := manager.Options{ + Scheme: scheme, + Logger: log.GetLogger(), + } + + mgr, err := manager.New(kConfig, managerOpts) + if err != nil { + return err + } + + //cl, err := client.New(kConfig, client.Options{ + // Scheme: scheme, + // WarningHandler: client.WarningHandlerOptions{}, + //}) + + schedulerCache := cache.NewCache(config.CacheSize) + log.Info("[subMain] scheduler cache was initialized") + log.Debug(fmt.Sprintf("[subMain] scheduler cache struct: %+v", schedulerCache)) - h, err := scheduler.NewHandler(ctx, cl, *log, config.DefaultDivisor) + h, err := scheduler.NewHandler(ctx, mgr.GetClient(), *log, schedulerCache, config.DefaultDivisor) if err != nil { return err } log.Info("[subMain] scheduler handler initialized") + _, err = controller.RunLVGWatcherCacheController(mgr, *log, schedulerCache) + if err != nil { + log.Error(err, fmt.Sprintf("[subMain] unable to run %s controller", controller.LVGWatcherCacheCtrlName)) + } + log.Info(fmt.Sprintf("[subMain] successfully ran %s controller", controller.LVGWatcherCacheCtrlName)) + + err = controller.RunPVCWatcherCacheController(ctx, mgr, *log, schedulerCache) + if err != nil { + log.Error(err, fmt.Sprintf("[subMain] unable to run %s controller", controller.PVCWatcherCacheCtrlName)) + } + log.Info(fmt.Sprintf("[subMain] successfully ran %s controller", controller.PVCWatcherCacheCtrlName)) + + if err = mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + log.Error(err, "[subMain] unable to mgr.AddHealthzCheck") + os.Exit(1) + } + log.Info("[subMain] successfully AddHealthzCheck") + + if err = mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + log.Error(err, "[subMain] unable to mgr.AddReadyzCheck") + os.Exit(1) + } + log.Info("[subMain] successfully AddReadyzCheck") + serv := &http.Server{ Addr: config.ListenAddr, Handler: accessLogHandler(parentCtx, h), ReadTimeout: 30 * time.Second, } + log.Info("[subMain] server was initialized") + var wg sync.WaitGroup defer wg.Wait() ctx, stop := signal.NotifyContext(parentCtx, os.Interrupt, syscall.SIGTERM) @@ -151,11 +196,22 @@ func subMain(parentCtx context.Context) error { } }() + go func() { + log.Info("[subMain] kube manager will start now") + err = mgr.Start(ctx) + if err != nil { + log.Error(err, "[subMain] unable to mgr.Start") + os.Exit(1) + } + }() + log.Info(fmt.Sprintf("[subMain] starts serving on: %s", config.ListenAddr)) err = serv.ListenAndServe() if !errors.Is(err, http.ErrServerClosed) { + log.Error(err, "[subMain] unable to run the server") return err } + return nil } diff --git a/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go b/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go new file mode 100644 index 00000000..71a58f01 --- /dev/null +++ b/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go @@ -0,0 +1,151 @@ +package cache + +import ( + "fmt" + v1 "k8s.io/api/core/v1" + "sds-local-volume-scheduler-extender/api/v1alpha1" +) + +const pvcPerLVGCount = 150 + +type Cache struct { + lvgs map[string]*lvgCache + pvcLVG map[string]string +} + +type lvgCache struct { + lvg *v1alpha1.LvmVolumeGroup + pvcs map[string]*pvcCache +} + +type pvcCache struct { + pvc *v1.PersistentVolumeClaim + nodeName string +} + +func NewCache(size int) *Cache { + return &Cache{ + lvgs: make(map[string]*lvgCache, size), + pvcLVG: make(map[string]string, size), + } +} + +func (c *Cache) AddLVG(lvg *v1alpha1.LvmVolumeGroup) error { + if _, exist := c.lvgs[lvg.Name]; exist { + return fmt.Errorf("the LVMVolumeGroup %s was found in the cache", lvg.Name) + } + + c.lvgs[lvg.Name] = &lvgCache{ + lvg: lvg, + pvcs: make(map[string]*pvcCache, pvcPerLVGCount), + } + + return nil +} + +func (c *Cache) UpdateLVG(lvg *v1alpha1.LvmVolumeGroup) error { + if cache, exist := c.lvgs[lvg.Name]; exist { + cache.lvg = lvg + return nil + } + + return fmt.Errorf("the LVMVolumeGroup %s was not found in the cache", lvg.Name) +} + +func (c *Cache) TryGetLVG(name string) *v1alpha1.LvmVolumeGroup { + if c.lvgs[name] == nil { + return nil + } + + return c.lvgs[name].lvg +} + +func (c *Cache) GetLVGNames() []string { + names := make([]string, 0, len(c.lvgs)) + for lvgName := range c.lvgs { + names = append(names, lvgName) + } + + return names +} + +func (c *Cache) GetLVGReservedSpace(lvgName string) int64 { + lvg := c.lvgs[lvgName] + + var space int64 + for _, pvc := range lvg.pvcs { + space += pvc.pvc.Spec.Resources.Requests.Storage().Value() + } + + return space +} + +func (c *Cache) DeleteLVG(lvgName string) { + delete(c.lvgs, lvgName) +} + +func (c *Cache) AddPVC(lvgName string, pvc *v1.PersistentVolumeClaim, pvcNodeName string) error { + if c.lvgs[lvgName].pvcs[pvc.Name] != nil { + return fmt.Errorf("PVC %s already exist in the cache", pvc.Name) + } + + c.lvgs[lvgName].pvcs[pvc.Name] = &pvcCache{pvc: pvc, nodeName: pvcNodeName} + c.pvcLVG[pvc.Name] = lvgName + return nil +} + +func (c *Cache) UpdatePVC(pvc *v1.PersistentVolumeClaim, pvcNodeName string) error { + lvgName := c.pvcLVG[pvc.Name] + if c.lvgs[lvgName].pvcs[pvc.Name] == nil { + return fmt.Errorf("PVC %s not found", pvc.Name) + } + + c.lvgs[lvgName].pvcs[pvc.Name].pvc = pvc + c.lvgs[lvgName].pvcs[pvc.Name].nodeName = pvcNodeName + return nil +} + +func (c *Cache) TryGetPVC(name string) *v1.PersistentVolumeClaim { + lvgName := c.pvcLVG[name] + + if c.lvgs[lvgName] == nil { + return nil + } + + return c.lvgs[lvgName].pvcs[name].pvc +} + +//func (c *Cache) GetCorrespondingLVGNameByPVC(pvcName string) string { +// return c.pvcLVG[pvcName] +//} + +func (c *Cache) GetAllPVCByLVG(lvgName string) []*v1.PersistentVolumeClaim { + pvcsCache := c.lvgs[lvgName] + + result := make([]*v1.PersistentVolumeClaim, 0, len(pvcsCache.pvcs)) + for _, cache := range pvcsCache.pvcs { + result = append(result, cache.pvc) + } + + return result +} + +func (c *Cache) GetPVCNodeName(pvcName string) string { + lvgName := c.pvcLVG[pvcName] + return c.lvgs[lvgName].pvcs[pvcName].nodeName +} + +func (c *Cache) GetAllPVCNames() []string { + result := make([]string, 0, len(c.pvcLVG)) + + for pvcName := range c.pvcLVG { + result = append(result, pvcName) + } + + return result +} + +func (c *Cache) RemovePVCSpaceReservation(pvcName string) { + lvgName := c.pvcLVG[pvcName] + delete(c.lvgs[lvgName].pvcs, pvcName) +} diff --git a/images/sds-local-volume-scheduler-extender/pkg/controller/lvg_watcher_cache.go b/images/sds-local-volume-scheduler-extender/pkg/controller/lvg_watcher_cache.go new file mode 100644 index 00000000..0fef0ac2 --- /dev/null +++ b/images/sds-local-volume-scheduler-extender/pkg/controller/lvg_watcher_cache.go @@ -0,0 +1,182 @@ +package controller + +import ( + "context" + "errors" + "fmt" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/client-go/util/workqueue" + "sds-local-volume-scheduler-extender/api/v1alpha1" + "sds-local-volume-scheduler-extender/pkg/cache" + "sds-local-volume-scheduler-extender/pkg/logger" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +const ( + LVGWatcherCacheCtrlName = "lvg-watcher-cache-controller" +) + +func RunLVGWatcherCacheController( + mgr manager.Manager, + log logger.Logger, + cache *cache.Cache, +) (controller.Controller, error) { + log.Info("[RunLVGWatcherCacheController] starts the work WITH EVENTS") + //cl := mgr.GetClient() + + c, err := controller.New(LVGWatcherCacheCtrlName, mgr, controller.Options{ + Reconciler: reconcile.Func(func(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + return reconcile.Result{}, nil + })}) + if err != nil { + log.Error(err, "[RunCacheWatcherController] unable to create a controller") + return nil, err + } + + err = c.Watch(source.Kind(mgr.GetCache(), &v1alpha1.LvmVolumeGroup{}), handler.Funcs{ + CreateFunc: func(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) { + log.Info(fmt.Sprintf("[RunLVGWatcherCacheController] CreateFunc starts the cache reconciliation for the LVMVolumeGroup %s", e.Object.GetName())) + + lvg, ok := e.Object.(*v1alpha1.LvmVolumeGroup) + if !ok { + err = errors.New("unable to cast event object to a given type") + log.Error(err, "[RunLVGWatcherCacheController] an error occurred while handling create event") + return + } + + if lvg.DeletionTimestamp != nil { + log.Info(fmt.Sprintf("[RunLVGWatcherCacheController] the LVMVolumeGroup %s should not be reconciled", lvg.Name)) + return + } + + // PopulateLVG + log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] tries to get the LVMVolumeGroup %s from the cache", lvg.Name)) + existedLVG := cache.TryGetLVG(lvg.Name) + + if existedLVG != nil { + log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] the LVMVolumeGroup %s was found. It will be updated", lvg.Name)) + err = cache.UpdateLVG(lvg) + if err != nil { + log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to update the LVMVolumeGroup %s cache", lvg.Name)) + } + + log.Info(fmt.Sprintf("[RunLVGWatcherCacheController] cache was updated for the LVMVolumeGroup %s", lvg.Name)) + } else { + log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] the LVMVolumeGroup %s was not found. It will be added", lvg.Name)) + err = cache.AddLVG(lvg) + if err != nil { + log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to add the LVMVolumeGroup %s to the cache", lvg.Name)) + } + + log.Info(fmt.Sprintf("[RunLVGWatcherCacheController] cache was added for the LVMVolumeGroup %s", lvg.Name)) + } + + //pvcs := &v1.PersistentVolumeClaimList{} + //err := cl.List(ctx, pvcs) + //if err != nil { + // log.Error(err, "[RunLVGWatcherCacheController] unable to list all PVCs") + // // TODO: requeue + // return + //} + // + //scsList := &v12.StorageClassList{} + //err = cl.List(ctx, scsList) + //if err != nil { + // log.Error(err, "[RunLVGWatcherCacheController] unable to list all StorageClasses") + // // TODO: requeue + // return + //} + //scs := make(map[string]v12.StorageClass, len(scsList.Items)) + //for _, sc := range scsList.Items { + // scs[sc.Name] = sc + //} + // + //lvgsBySC, err := scheduler.GetSortedLVGsFromStorageClasses(scs) + //if err != nil { + // log.Error(err, "[RunLVGWatcherCacheController] unable to sort LVGs by StorageClasses") + // // TODO: requeue + // return + //} + + pvcs := cache.GetAllPVCByLVG(lvg.Name) + for _, pvc := range pvcs { + if pvc.Status.Phase == v1.ClaimBound { + cache.RemovePVCSpaceReservation(pvc.Name) + } + } + + log.Info(fmt.Sprintf("[RunLVGWatcherCacheController] cache for the LVMVolumeGroup %s was reconciled by CreateFunc", lvg.Name)) + }, + UpdateFunc: func(ctx context.Context, e event.UpdateEvent, q workqueue.RateLimitingInterface) { + log.Info(fmt.Sprintf("[RunCacheWatcherController] UpdateFunc starts the cache reconciliation for the LVMVolumeGroup %s", e.ObjectNew.GetName())) + oldLvg, ok := e.ObjectOld.(*v1alpha1.LvmVolumeGroup) + if !ok { + err = errors.New("unable to cast event object to a given type") + log.Error(err, "[RunLVGWatcherCacheController] an error occurred while handling create event") + return + } + + newLvg, ok := e.ObjectNew.(*v1alpha1.LvmVolumeGroup) + if !ok { + err = errors.New("unable to cast event object to a given type") + log.Error(err, "[RunLVGWatcherCacheController] an error occurred while handling create event") + return + } + + oldSize, err := resource.ParseQuantity(oldLvg.Status.AllocatedSize) + if err != nil { + log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to parse the allocated size for the LVMVolumeGroup %s", oldLvg.Name)) + return + } + + newSize, err := resource.ParseQuantity(newLvg.Status.AllocatedSize) + if err != nil { + log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to parse the allocated size for the LVMVolumeGroup %s", oldLvg.Name)) + return + } + + if newLvg.DeletionTimestamp != nil || + oldSize.Value() == newSize.Value() { + log.Info(fmt.Sprintf("[RunLVGWatcherCacheController] the LVMVolumeGroup %s should not be reconciled", newLvg.Name)) + return + } + + err = cache.UpdateLVG(newLvg) + if err != nil { + log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to update the LVMVolumeGroup %s cache", newLvg.Name)) + } + + pvcs := cache.GetAllPVCByLVG(newLvg.Name) + for _, pvc := range pvcs { + if pvc.Status.Phase == v1.ClaimBound { + cache.RemovePVCSpaceReservation(pvc.Name) + } + } + + log.Info(fmt.Sprintf("[RunLVGWatcherCacheController] updated LVMVolumeGroup %s cache size", newLvg.Name)) + }, + DeleteFunc: func(ctx context.Context, e event.DeleteEvent, q workqueue.RateLimitingInterface) { + log.Info(fmt.Sprintf("[RunCacheWatcherController] DeleteFunc starts the cache reconciliation for the LVMVolumeGroup %s", e.Object.GetName())) + lvg, ok := e.Object.(*v1alpha1.LvmVolumeGroup) + if !ok { + err = errors.New("unable to cast event object to a given type") + log.Error(err, "[RunLVGWatcherCacheController] an error occurred while handling create event") + return + } + cache.DeleteLVG(lvg.Name) + log.Info(fmt.Sprintf("[RunLVGWatcherCacheController] LVMVolumeGroup %s was deleted from the cache", lvg.Name)) + }, + }) + if err != nil { + log.Error(err, "[RunCacheWatcherController] unable to watch the events") + return nil, err + } + + return c, nil +} diff --git a/images/sds-local-volume-scheduler-extender/pkg/controller/pvc_watcher_cache.go b/images/sds-local-volume-scheduler-extender/pkg/controller/pvc_watcher_cache.go new file mode 100644 index 00000000..4e25b410 --- /dev/null +++ b/images/sds-local-volume-scheduler-extender/pkg/controller/pvc_watcher_cache.go @@ -0,0 +1,174 @@ +package controller + +import ( + "context" + "errors" + "fmt" + v1 "k8s.io/api/core/v1" + v12 "k8s.io/api/storage/v1" + cache2 "k8s.io/client-go/tools/cache" + "sds-local-volume-scheduler-extender/api/v1alpha1" + "sds-local-volume-scheduler-extender/pkg/cache" + "sds-local-volume-scheduler-extender/pkg/logger" + "sds-local-volume-scheduler-extender/pkg/scheduler" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +const ( + PVCWatcherCacheCtrlName = "pvc-watcher-cache-controller" + selectedNodeAnnotation = "volume.kubernetes.io/selected-node" +) + +func RunPVCWatcherCacheController( + ctx context.Context, + mgr manager.Manager, + log logger.Logger, + schedulerCache *cache.Cache, +) error { + log.Info("[RunPVCWatcherCacheController] starts the work WITH EVENTS") + + cl := mgr.GetClient() + inf, err := mgr.GetCache().GetInformer(ctx, &v1.PersistentVolumeClaim{}) + if err != nil { + log.Error(err, "[RunPVCWatcherCacheController] unable to get the informer") + return err + } + + inf.AddEventHandler(cache2.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + pvc, ok := obj.(*v1.PersistentVolumeClaim) + if !ok { + err = errors.New("unable to cast event object to a given type") + log.Error(err, "[RunPVCWatcherCacheController] an error occurred while handling create event") + } + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] AddFunc starts the reconciliation for the PVC %s", pvc.Name)) + + if shouldReconcilvePVC(pvc) { + cachedPvc := schedulerCache.TryGetPVC(pvc.Name) + if cachedPvc != nil { + if pvc.Status.Phase == v1.ClaimBound { + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s in a Bound state. It will be removed from the cache")) + //schedulerCache.RemovePVCSpaceReservation(pvc.Name) + log.Info(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s was removed from the cache")) + return + } + + err := schedulerCache.UpdatePVC(pvc, pvc.Annotations[selectedNodeAnnotation]) + if err != nil { + log.Error(err, fmt.Sprintf("[RunPVCWatcherCacheController] unable to update PVC %s in the cache", pvc.Name)) + } + } else { + lvg, err := findLVGByPVC(ctx, cl, pvc) + if err != nil { + log.Error(err, "[RunPVCWatcherCacheController] an error occurs the founding a LVMVolumeGroup") + // TODO: requeue or something + return + } + + err = schedulerCache.AddPVC(lvg.Name, pvc, pvc.Annotations[selectedNodeAnnotation]) + if err != nil { + log.Error(err, "[RunPVCWatcherCacheController] unable to add PVC to the cache") + // TODO: requeue or something + return + } + } + } + + }, + UpdateFunc: func(oldObj, newObj interface{}) { + pvc, ok := newObj.(*v1.PersistentVolumeClaim) + if !ok { + err = errors.New("unable to cast event object to a given type") + log.Error(err, "[RunPVCWatcherCacheController] an error occurred while handling create event") + } + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] AddFunc starts the reconciliation for the PVC %s", pvc.Name)) + + if shouldReconcilvePVC(pvc) { + cachedPvc := schedulerCache.TryGetPVC(pvc.Name) + if cachedPvc != nil { + if pvc.Status.Phase == v1.ClaimBound { + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s in a Bound state. It will be removed from the cache")) + //schedulerCache.RemovePVCSpaceReservation(pvc.Name) + log.Info(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s was removed from the cache")) + return + } + + err := schedulerCache.UpdatePVC(pvc, pvc.Annotations[selectedNodeAnnotation]) + if err != nil { + log.Error(err, fmt.Sprintf("[RunPVCWatcherCacheController] unable to update PVC %s in the cache", pvc.Name)) + } + } else { + lvg, err := findLVGByPVC(ctx, cl, pvc) + if err != nil { + log.Error(err, "[RunPVCWatcherCacheController] an error occurs the founding a LVMVolumeGroup") + // TODO: requeue or something + return + } + + err = schedulerCache.AddPVC(lvg.Name, pvc, pvc.Annotations[selectedNodeAnnotation]) + if err != nil { + log.Error(err, "[RunPVCWatcherCacheController] unable to add PVC to the cache") + // TODO: requeue or something + return + } + } + } + }, + DeleteFunc: func(obj interface{}) { + log.Info("HELLO FROM DELETEFUNC") + }, + }) + + return nil +} + +func shouldReconcilvePVC(pvc *v1.PersistentVolumeClaim) bool { + _, selected := pvc.Annotations[selectedNodeAnnotation] + if pvc.Status.Phase != v1.ClaimBound && selected { + return true + } + + return false +} + +func findLVGByPVC(ctx context.Context, cl client.Client, pvc *v1.PersistentVolumeClaim) (*v1alpha1.LvmVolumeGroup, error) { + sc := &v12.StorageClass{} + // TODO: Будет ли проставлен storage class для PVC, если не будет указан явно (иначе зачем тут поинтер?) + err := cl.Get(ctx, client.ObjectKey{ + Name: *pvc.Spec.StorageClassName, + }, sc) + if err != nil { + return nil, fmt.Errorf("[findLVGByPVC] unable to get a storage class %s", *pvc.Spec.StorageClassName) + } + + lvgsFromSC, err := scheduler.ExtractLVGsFromSC(*sc) + + lvgList := &v1alpha1.LvmVolumeGroupList{} + err = cl.List(ctx, lvgList) + if err != nil { + return nil, fmt.Errorf("[findLVGByPVC] unable to list LVMVolumeGroups") + } + + lvgs := make(map[string]v1alpha1.LvmVolumeGroup, len(lvgList.Items)) + for _, lvg := range lvgList.Items { + lvgs[lvg.Name] = lvg + } + + for _, lvg := range lvgsFromSC { + kubeLVG, exist := lvgs[lvg.Name] + if !exist { + return nil, fmt.Errorf("unable to found the LVMVolumeGroup %s for storage class %s", lvg.Name, sc.Name) + } + + if kubeLVG.Status.Nodes == nil || len(kubeLVG.Status.Nodes) == 0 { + return nil, fmt.Errorf("no nodes specified for the LVMVolumeGroup %s for storage class %s", lvg.Name, sc.Name) + } + + if kubeLVG.Status.Nodes[0].Name == pvc.Annotations[selectedNodeAnnotation] { + return &kubeLVG, nil + } + } + + return nil, nil +} diff --git a/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go b/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go index 09195373..27d4000b 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go +++ b/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go @@ -41,7 +41,7 @@ const ( thin = "Thin" ) -func (s scheduler) filter(w http.ResponseWriter, r *http.Request) { +func (s *scheduler) filter(w http.ResponseWriter, r *http.Request) { s.log.Debug("[filter] starts the serving") var input ExtenderArgs reader := http.MaxBytesReader(w, r.Body, 10<<20) @@ -176,7 +176,7 @@ func filterNodes( }, nil } - lvgs, err := getLVMVolumeGroups(ctx, cl) + lvgs, err := GetLVMVolumeGroups(ctx, cl) if err != nil { return nil, err } @@ -188,14 +188,14 @@ func filterNodes( log.Trace(fmt.Sprintf("[filterNodes] LVGs Thick FreeSpace: %+v", lvgsThickFree)) lvgsThickFreeMutex := &sync.RWMutex{} - scLVGs, err := getSortedLVGsFromStorageClasses(scs) + scLVGs, err := GetSortedLVGsFromStorageClasses(scs) if err != nil { return nil, err } - usedLVGs := removeUnusedLVGs(lvgs, scLVGs) + usedLVGs := RemoveUnusedLVGs(lvgs, scLVGs) - nodeLVGs := sortLVGsByNodeName(usedLVGs) + nodeLVGs := SortLVGsByNodeName(usedLVGs) for n, ls := range nodeLVGs { for _, l := range ls { log.Trace(fmt.Sprintf("[filterNodes] the LVMVolumeGroup %s belongs to node %s", l.Name, n)) @@ -374,7 +374,7 @@ func getCommonNodesByStorageClasses(scs map[string]v1.StorageClass, nodesWithLVG nodeIncludesLVG := true for _, sc := range scs { - scLvgs, err := extractLVGsFromSC(sc) + scLvgs, err := ExtractLVGsFromSC(sc) if err != nil { return nil, err } @@ -401,7 +401,7 @@ func getCommonNodesByStorageClasses(scs map[string]v1.StorageClass, nodesWithLVG return result, nil } -func removeUnusedLVGs(lvgs map[string]v1alpha1.LvmVolumeGroup, scsLVGs map[string]LVMVolumeGroups) map[string]v1alpha1.LvmVolumeGroup { +func RemoveUnusedLVGs(lvgs map[string]v1alpha1.LvmVolumeGroup, scsLVGs map[string]LVMVolumeGroups) map[string]v1alpha1.LvmVolumeGroup { result := make(map[string]v1alpha1.LvmVolumeGroup, len(lvgs)) usedLvgs := make(map[string]struct{}, len(lvgs)) @@ -420,11 +420,11 @@ func removeUnusedLVGs(lvgs map[string]v1alpha1.LvmVolumeGroup, scsLVGs map[strin return result } -func getSortedLVGsFromStorageClasses(scs map[string]v1.StorageClass) (map[string]LVMVolumeGroups, error) { +func GetSortedLVGsFromStorageClasses(scs map[string]v1.StorageClass) (map[string]LVMVolumeGroups, error) { result := make(map[string]LVMVolumeGroups, len(scs)) for _, sc := range scs { - lvgs, err := extractLVGsFromSC(sc) + lvgs, err := ExtractLVGsFromSC(sc) if err != nil { return nil, err } @@ -445,7 +445,7 @@ type LVMVolumeGroup struct { } type LVMVolumeGroups []LVMVolumeGroup -func extractLVGsFromSC(sc v1.StorageClass) (LVMVolumeGroups, error) { +func ExtractLVGsFromSC(sc v1.StorageClass) (LVMVolumeGroups, error) { var lvmVolumeGroups LVMVolumeGroups err := yaml.Unmarshal([]byte(sc.Parameters[lvmVolumeGroupsParamKey]), &lvmVolumeGroups) if err != nil { @@ -454,7 +454,7 @@ func extractLVGsFromSC(sc v1.StorageClass) (LVMVolumeGroups, error) { return lvmVolumeGroups, nil } -func sortLVGsByNodeName(lvgs map[string]v1alpha1.LvmVolumeGroup) map[string][]v1alpha1.LvmVolumeGroup { +func SortLVGsByNodeName(lvgs map[string]v1alpha1.LvmVolumeGroup) map[string][]v1alpha1.LvmVolumeGroup { sorted := make(map[string][]v1alpha1.LvmVolumeGroup, len(lvgs)) for _, lvg := range lvgs { for _, node := range lvg.Status.Nodes { @@ -465,7 +465,7 @@ func sortLVGsByNodeName(lvgs map[string]v1alpha1.LvmVolumeGroup) map[string][]v1 return sorted } -func getLVMVolumeGroups(ctx context.Context, cl client.Client) (map[string]v1alpha1.LvmVolumeGroup, error) { +func GetLVMVolumeGroups(ctx context.Context, cl client.Client) (map[string]v1alpha1.LvmVolumeGroup, error) { lvgl := &v1alpha1.LvmVolumeGroupList{} err := cl.List(ctx, lvgl) if err != nil { diff --git a/images/sds-local-volume-scheduler-extender/pkg/scheduler/prioritize.go b/images/sds-local-volume-scheduler-extender/pkg/scheduler/prioritize.go index c966f6c7..77a59b35 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/scheduler/prioritize.go +++ b/images/sds-local-volume-scheduler-extender/pkg/scheduler/prioritize.go @@ -32,7 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -func (s scheduler) prioritize(w http.ResponseWriter, r *http.Request) { +func (s *scheduler) prioritize(w http.ResponseWriter, r *http.Request) { s.log.Debug("[prioritize] starts serving") var input ExtenderArgs reader := http.MaxBytesReader(w, r.Body, 10<<20) @@ -99,22 +99,22 @@ func scoreNodes( pvcRequests map[string]PVCRequest, divisor float64, ) ([]HostPriority, error) { - lvgs, err := getLVMVolumeGroups(ctx, cl) + lvgs, err := GetLVMVolumeGroups(ctx, cl) if err != nil { return nil, err } - scLVGs, err := getSortedLVGsFromStorageClasses(scs) + scLVGs, err := GetSortedLVGsFromStorageClasses(scs) if err != nil { return nil, err } - usedLVGs := removeUnusedLVGs(lvgs, scLVGs) + usedLVGs := RemoveUnusedLVGs(lvgs, scLVGs) for lvgName := range usedLVGs { log.Trace(fmt.Sprintf("[scoreNodes] used LVMVolumeGroup %s", lvgName)) } - nodeLVGs := sortLVGsByNodeName(usedLVGs) + nodeLVGs := SortLVGsByNodeName(usedLVGs) for n, ls := range nodeLVGs { for _, l := range ls { log.Trace(fmt.Sprintf("[scoreNodes] the LVMVolumeGroup %s belongs to node %s", l.Name, n)) diff --git a/images/sds-local-volume-scheduler-extender/pkg/scheduler/route.go b/images/sds-local-volume-scheduler-extender/pkg/scheduler/route.go index 4176a928..b5d1951a 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/scheduler/route.go +++ b/images/sds-local-volume-scheduler-extender/pkg/scheduler/route.go @@ -20,8 +20,8 @@ import ( "context" "fmt" "net/http" + "sds-local-volume-scheduler-extender/pkg/cache" "sds-local-volume-scheduler-extender/pkg/logger" - "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -30,9 +30,10 @@ type scheduler struct { log logger.Logger client client.Client ctx context.Context + cache *cache.Cache } -func (s scheduler) ServeHTTP(w http.ResponseWriter, r *http.Request) { +func (s *scheduler) ServeHTTP(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/filter": s.log.Debug("[ServeHTTP] filter route starts handling the request") @@ -46,18 +47,23 @@ func (s scheduler) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.log.Debug("[ServeHTTP] status route starts handling the request") status(w, r) s.log.Debug("[ServeHTTP] status route ends handling the request") + case "/cache": + s.log.Debug("[ServeHTTP] cache route starts handling the request") + s.getCache(w, r) + s.log.Debug("[ServeHTTP] cache route ends handling the request") default: http.Error(w, "not found", http.StatusNotFound) } } // NewHandler return new http.Handler of the scheduler extender -func NewHandler(ctx context.Context, cl client.Client, log logger.Logger, defaultDiv float64) (http.Handler, error) { - return scheduler{ +func NewHandler(ctx context.Context, cl client.Client, log logger.Logger, lvgCache *cache.Cache, defaultDiv float64) (http.Handler, error) { + return &scheduler{ defaultDivisor: defaultDiv, log: log, client: cl, ctx: ctx, + cache: lvgCache, }, nil } @@ -68,3 +74,38 @@ func status(w http.ResponseWriter, r *http.Request) { fmt.Println(fmt.Sprintf("error occurs on status route, err: %s", err.Error())) } } + +func (s *scheduler) getCache(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + + result := make(map[string][]struct { + pvcName string + nodeName string + }) + + lvgs := s.cache.GetLVGNames() + s.log.Info(fmt.Sprintf("LVG from cache: %v", lvgs)) + for _, lvg := range lvgs { + pvcs := s.cache.GetAllPVCByLVG(lvg) + s.log.Info(fmt.Sprintf("LVG %s has PVC from cache: %v", lvg, pvcs)) + + result[lvg] = make([]struct { + pvcName string + nodeName string + }, 0) + + for _, pvc := range pvcs { + result[lvg] = append(result[lvg], struct { + pvcName string + nodeName string + }{pvcName: pvc.Name, nodeName: s.cache.GetPVCNodeName(pvc.Name)}) + } + } + s.log.Info(fmt.Sprintf("Result len: %d", len(result))) + + _, err := w.Write([]byte(fmt.Sprintf("%+v", result))) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("unable to write the cache")) + } +} diff --git a/templates/sds-local-volume-scheduler-extender/rbac-for-us.yaml b/templates/sds-local-volume-scheduler-extender/rbac-for-us.yaml index 316b9857..b2b7fa07 100644 --- a/templates/sds-local-volume-scheduler-extender/rbac-for-us.yaml +++ b/templates/sds-local-volume-scheduler-extender/rbac-for-us.yaml @@ -46,7 +46,10 @@ rules: verbs: ["create", "get", "update"] - apiGroups: [ "storage.deckhouse.io" ] resources: [ "lvmvolumegroups" ] - verbs: [ "list" ] + verbs: [ "list", "watch"] + - apiGroups: ["v1"] + resources: ["persistentvolumeclaims"] + verbs: ["list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding From c436bc160c1c17a6e7143fcb751ddb0acbeb6789 Mon Sep 17 00:00:00 2001 From: Aleksandr Zimin Date: Sun, 31 Mar 2024 22:30:34 +0300 Subject: [PATCH 02/21] add log levels to csi Signed-off-by: Aleksandr Zimin --- .../sds-local-volume-csi/controller.yaml | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/templates/sds-local-volume-csi/controller.yaml b/templates/sds-local-volume-csi/controller.yaml index 2419552f..e405eff3 100644 --- a/templates/sds-local-volume-csi/controller.yaml +++ b/templates/sds-local-volume-csi/controller.yaml @@ -233,6 +233,18 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName + - name: LOG_LEVEL + {{- if eq .Values.sdsLocalVolume.logLevel "ERROR" }} + value: "0" + {{- else if eq .Values.sdsLocalVolume.logLevel "WARN" }} + value: "1" + {{- else if eq .Values.sdsLocalVolume.logLevel "INFO" }} + value: "2" + {{- else if eq .Values.sdsLocalVolume.logLevel "DEBUG" }} + value: "3" + {{- else if eq .Values.sdsLocalVolume.logLevel "TRACE" }} + value: "4" + {{- end }} image: {{ include "helm_lib_module_image" (list . "sdsLocalVolumeCsi") }} imagePullPolicy: IfNotPresent livenessProbe: @@ -511,6 +523,18 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName + - name: LOG_LEVEL + {{- if eq .Values.sdsLocalVolume.logLevel "ERROR" }} + value: "0" + {{- else if eq .Values.sdsLocalVolume.logLevel "WARN" }} + value: "1" + {{- else if eq .Values.sdsLocalVolume.logLevel "INFO" }} + value: "2" + {{- else if eq .Values.sdsLocalVolume.logLevel "DEBUG" }} + value: "3" + {{- else if eq .Values.sdsLocalVolume.logLevel "TRACE" }} + value: "4" + {{- end }} image: {{ include "helm_lib_module_image" (list . "sdsLocalVolumeCsi") }} imagePullPolicy: IfNotPresent livenessProbe: From d9c1ef8c85c0ed2ad1247c71f3fc3227ea455c6e Mon Sep 17 00:00:00 2001 From: Aleksandr Zimin Date: Sun, 31 Mar 2024 23:24:11 +0300 Subject: [PATCH 03/21] Fix Signed-off-by: Aleksandr Zimin --- images/sds-local-volume-csi/driver/node.go | 9 +++- .../sds-local-volume-csi/pkg/utils/volume.go | 43 +++++++++++++------ 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/images/sds-local-volume-csi/driver/node.go b/images/sds-local-volume-csi/driver/node.go index 4575a4dc..71bb3b68 100644 --- a/images/sds-local-volume-csi/driver/node.go +++ b/images/sds-local-volume-csi/driver/node.go @@ -37,12 +37,13 @@ func (d *Driver) NodeUnstageVolume(ctx context.Context, request *csi.NodeUnstage } func (d *Driver) NodePublishVolume(ctx context.Context, request *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { - d.log.Info("method NodePublishVolume") + d.log.Info("Start method NodePublishVolume") d.log.Trace("------------- NodePublishVolume --------------") d.log.Trace(request.String()) d.log.Trace("------------- NodePublishVolume --------------") dev := fmt.Sprintf("/dev/%s/%s", request.GetVolumeContext()[internal.VGNameKey], request.VolumeId) + d.log.Info("dev = ", dev) var mountOptions []string if request.GetReadonly() { @@ -61,12 +62,18 @@ func (d *Driver) NodePublishVolume(ctx context.Context, request *csi.NodePublish mountOptions = append(mountOptions, mnt.GetMountFlags()...) } + d.log.Info("mountOptions = ", mountOptions) + d.log.Info("fsType = ", fsType) + d.log.Info("IsBlock = ", IsBlock) + err := d.storeManager.Mount(dev, request.GetTargetPath(), IsBlock, fsType, false, mountOptions) if err != nil { d.log.Error(err, "d.mounter.Mount :") return nil, err } + d.log.Info("Success method NodePublishVolume") + return &csi.NodePublishVolumeResponse{}, nil } diff --git a/images/sds-local-volume-csi/pkg/utils/volume.go b/images/sds-local-volume-csi/pkg/utils/volume.go index c4dec3e8..bd979642 100644 --- a/images/sds-local-volume-csi/pkg/utils/volume.go +++ b/images/sds-local-volume-csi/pkg/utils/volume.go @@ -20,7 +20,6 @@ import ( "fmt" "os" "sds-local-volume-csi/pkg/logger" - "time" mu "k8s.io/mount-utils" utilexec "k8s.io/utils/exec" @@ -30,6 +29,7 @@ type NodeStoreManager interface { Mount(source, target string, isBlock bool, fsType string, readonly bool, mntOpts []string) error Unmount(target string) error IsNotMountPoint(target string) (bool, error) + IsMountPointCorrect(mountRefs []string, target string) bool ResizeFS(target string) error } @@ -77,23 +77,34 @@ func (s *Store) Mount(source, target string, isBlock bool, fsType string, readon s.Log.Trace("-----------------== start MkdirAll ==-----------------") s.Log.Trace("mkdir create dir =" + target) if err := os.MkdirAll(target, os.FileMode(0755)); err != nil { - return fmt.Errorf("[MkdirAll] could not create target directory %s, %v", target, err) + return fmt.Errorf("[MkdirAll] could not create target directory %s: %w", target, err) } s.Log.Trace("-----------------== stop MkdirAll ==-----------------") - needsMount, err := s.NodeStorage.IsMountPoint(target) + isMountPoint, err := s.NodeStorage.IsMountPoint(target) if err != nil { - return fmt.Errorf("[s.NodeStorage.IsMountPoint] unable to determine mount status of %s %v", target, err) + return fmt.Errorf("[s.NodeStorage.IsMountPoint] unable to determine mount status of %s: %w", target, err) } - s.Log.Trace("≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈ needsMount ≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈") - s.Log.Trace(fmt.Sprintf("%t", needsMount)) - s.Log.Trace("≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈ needsMount ≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈") + s.Log.Trace("≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈ isMountPoint ≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈") + s.Log.Trace(fmt.Sprintf("%t", isMountPoint)) + s.Log.Trace("≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈ isMountPoint ≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈") - //todo - //if !needsMount { - // return nil - //} + if isMountPoint { + s.Log.Trace("Try to find mount refs for source %s", source) + mountRefs, err := s.NodeStorage.GetMountRefs(source) + if err != nil { + return fmt.Errorf("failed to get mount refs for source %s: %w", source, err) + } + s.Log.Trace("Found mount refs: %v", mountRefs) + mountPointCorrect := s.IsMountPointCorrect(mountRefs, target) + if !mountPointCorrect { + return fmt.Errorf("target %s is a mount point and is not mounted to source %s", target, source) + } + + s.Log.Trace("Target %s is a mount point and already mounted to source %s. Skipping FormatAndMount without any checks", target, source) + return nil + } s.Log.Trace("-----------------== start FormatAndMount ==---------------") err = s.NodeStorage.FormatAndMount(source, target, fsType, mntOpts) @@ -138,7 +149,6 @@ func (s *Store) Unmount(target string) error { s.Log.Error(err, "[s.NodeStorage.Unmount]: ") return err } - time.Sleep(time.Second * 1) return nil } @@ -172,3 +182,12 @@ func (s *Store) ResizeFS(mountTarget string) error { s.Log.Info("Filesystem resized successfully", "devicePath", devicePath) return nil } + +func (s *Store) IsMountPointCorrect(mountRefs []string, target string) bool { + for _, ref := range mountRefs { + if ref == target { + return true + } + } + return false +} From e1deb1814d157fcc308788c6be9816815e05fae6 Mon Sep 17 00:00:00 2001 From: Aleksandr Zimin Date: Sun, 31 Mar 2024 23:56:42 +0300 Subject: [PATCH 04/21] fix log Signed-off-by: Aleksandr Zimin --- images/sds-local-volume-csi/driver/node.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/images/sds-local-volume-csi/driver/node.go b/images/sds-local-volume-csi/driver/node.go index 71bb3b68..22f850a0 100644 --- a/images/sds-local-volume-csi/driver/node.go +++ b/images/sds-local-volume-csi/driver/node.go @@ -43,7 +43,7 @@ func (d *Driver) NodePublishVolume(ctx context.Context, request *csi.NodePublish d.log.Trace("------------- NodePublishVolume --------------") dev := fmt.Sprintf("/dev/%s/%s", request.GetVolumeContext()[internal.VGNameKey], request.VolumeId) - d.log.Info("dev = ", dev) + d.log.Info(fmt.Sprintf("dev = %s", dev)) var mountOptions []string if request.GetReadonly() { @@ -62,9 +62,9 @@ func (d *Driver) NodePublishVolume(ctx context.Context, request *csi.NodePublish mountOptions = append(mountOptions, mnt.GetMountFlags()...) } - d.log.Info("mountOptions = ", mountOptions) - d.log.Info("fsType = ", fsType) - d.log.Info("IsBlock = ", IsBlock) + d.log.Info(fmt.Sprintf("mountOptions = %s", mountOptions)) + d.log.Info(fmt.Sprintf("fsType = %s", fsType)) + d.log.Info(fmt.Sprintf("IsBlock = %t", IsBlock)) err := d.storeManager.Mount(dev, request.GetTargetPath(), IsBlock, fsType, false, mountOptions) if err != nil { From dcfa0fdbf9219c2b09c2b6d780c2f5bc3d692a1e Mon Sep 17 00:00:00 2001 From: Aleksandr Zimin Date: Mon, 1 Apr 2024 00:00:15 +0300 Subject: [PATCH 05/21] fix loggin2 Signed-off-by: Aleksandr Zimin --- images/sds-local-volume-csi/pkg/utils/volume.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/images/sds-local-volume-csi/pkg/utils/volume.go b/images/sds-local-volume-csi/pkg/utils/volume.go index bd979642..b3a6f440 100644 --- a/images/sds-local-volume-csi/pkg/utils/volume.go +++ b/images/sds-local-volume-csi/pkg/utils/volume.go @@ -91,22 +91,23 @@ func (s *Store) Mount(source, target string, isBlock bool, fsType string, readon s.Log.Trace("≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈ isMountPoint ≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈") if isMountPoint { - s.Log.Trace("Try to find mount refs for source %s", source) + s.Log.Trace(fmt.Sprintf("Try to find mount refs for source %s", source)) mountRefs, err := s.NodeStorage.GetMountRefs(source) if err != nil { return fmt.Errorf("failed to get mount refs for source %s: %w", source, err) } - s.Log.Trace("Found mount refs: %v", mountRefs) + s.Log.Trace(fmt.Sprintf("Found mount refs: %v", mountRefs)) mountPointCorrect := s.IsMountPointCorrect(mountRefs, target) if !mountPointCorrect { return fmt.Errorf("target %s is a mount point and is not mounted to source %s", target, source) } - s.Log.Trace("Target %s is a mount point and already mounted to source %s. Skipping FormatAndMount without any checks", target, source) + s.Log.Trace(fmt.Sprintf("Target %s is a mount point and already mounted to source %s. Skipping FormatAndMount without any checks", target, source)) return nil } s.Log.Trace("-----------------== start FormatAndMount ==---------------") + err = s.NodeStorage.FormatAndMount(source, target, fsType, mntOpts) if err != nil { return fmt.Errorf("failed to FormatAndMount : %w", err) From 6cf823925e61d692a1a5430ee88df394546ff7fd Mon Sep 17 00:00:00 2001 From: Aleksandr Zimin Date: Mon, 1 Apr 2024 01:49:04 +0300 Subject: [PATCH 06/21] fix Signed-off-by: Aleksandr Zimin --- .../sds-local-volume-csi/driver/controller.go | 19 ++++--- images/sds-local-volume-csi/driver/node.go | 8 ++- images/sds-local-volume-csi/internal/const.go | 1 + .../sds-local-volume-csi/pkg/utils/volume.go | 52 +++++++++++-------- 4 files changed, 49 insertions(+), 31 deletions(-) diff --git a/images/sds-local-volume-csi/driver/controller.go b/images/sds-local-volume-csi/driver/controller.go index 5d94f974..02dabcb9 100644 --- a/images/sds-local-volume-csi/driver/controller.go +++ b/images/sds-local-volume-csi/driver/controller.go @@ -66,12 +66,12 @@ func (d *Driver) CreateVolume(ctx context.Context, request *csi.CreateVolumeRequ d.log.Error(err, "error GetStorageClassLVGs") return nil, status.Errorf(codes.Internal, err.Error()) } -// TODO: Consider refactoring the naming strategy for llvName and lvName. -// Currently, we use the same name for llvName (the name of the LVMLogicalVolume resource in Kubernetes) -// and lvName (the name of the LV in LVM on the node) because the PV name is unique within the cluster, -// preventing name collisions. This approach simplifies matching between nodes and Kubernetes by maintaining -// the same name in both contexts. Future consideration should be given to optimizing this logic to enhance -// code readability and maintainability. + // TODO: Consider refactoring the naming strategy for llvName and lvName. + // Currently, we use the same name for llvName (the name of the LVMLogicalVolume resource in Kubernetes) + // and lvName (the name of the LV in LVM on the node) because the PV name is unique within the cluster, + // preventing name collisions. This approach simplifies matching between nodes and Kubernetes by maintaining + // the same name in both contexts. Future consideration should be given to optimizing this logic to enhance + // code readability and maintainability. llvName := request.Name lvName := request.Name d.log.Info(fmt.Sprintf("llv name: %s ", llvName)) @@ -149,7 +149,12 @@ func (d *Driver) CreateVolume(ctx context.Context, request *csi.CreateVolumeRequ } volumeCtx[internal.SubPath] = request.Name - volumeCtx[internal.VGNameKey] = selectedLVG.Spec.ActualVGNameOnTheNode + volumeCtx[internal.VGNameKey] = llvSpec.LvmVolumeGroupName + if llvSpec.Type == internal.LLMTypeThin { + volumeCtx[internal.ThinPoolNameKey] = llvSpec.Thin.PoolName + } else { + volumeCtx[internal.ThinPoolNameKey] = "" + } return &csi.CreateVolumeResponse{ Volume: &csi.Volume{ diff --git a/images/sds-local-volume-csi/driver/node.go b/images/sds-local-volume-csi/driver/node.go index 22f850a0..cb08c4b2 100644 --- a/images/sds-local-volume-csi/driver/node.go +++ b/images/sds-local-volume-csi/driver/node.go @@ -43,6 +43,8 @@ func (d *Driver) NodePublishVolume(ctx context.Context, request *csi.NodePublish d.log.Trace("------------- NodePublishVolume --------------") dev := fmt.Sprintf("/dev/%s/%s", request.GetVolumeContext()[internal.VGNameKey], request.VolumeId) + lvmType := request.GetVolumeContext()[internal.LvmTypeKey] + lvmThinPoolName := request.GetVolumeContext()[internal.ThinPoolNameKey] d.log.Info(fmt.Sprintf("dev = %s", dev)) var mountOptions []string @@ -63,10 +65,12 @@ func (d *Driver) NodePublishVolume(ctx context.Context, request *csi.NodePublish } d.log.Info(fmt.Sprintf("mountOptions = %s", mountOptions)) + d.log.Info(fmt.Sprintf("lvmType = %s", lvmType)) + d.log.Info(fmt.Sprintf("lvmThinPoolName = %s", lvmThinPoolName)) d.log.Info(fmt.Sprintf("fsType = %s", fsType)) d.log.Info(fmt.Sprintf("IsBlock = %t", IsBlock)) - err := d.storeManager.Mount(dev, request.GetTargetPath(), IsBlock, fsType, false, mountOptions) + err := d.storeManager.Mount(dev, request.GetTargetPath(), IsBlock, fsType, false, mountOptions, lvmType, lvmThinPoolName) if err != nil { d.log.Error(err, "d.mounter.Mount :") return nil, err @@ -146,7 +150,7 @@ func (d *Driver) NodeGetCapabilities(ctx context.Context, request *csi.NodeGetCa func (d *Driver) NodeGetInfo(ctx context.Context, request *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) { d.log.Info("method NodeGetInfo") - d.log.Info("hostID = ", d.hostID) + d.log.Info(fmt.Sprintf("hostID = %s", d.hostID)) return &csi.NodeGetInfoResponse{ NodeId: d.hostID, diff --git a/images/sds-local-volume-csi/internal/const.go b/images/sds-local-volume-csi/internal/const.go index fd2506fc..9bd59a5e 100644 --- a/images/sds-local-volume-csi/internal/const.go +++ b/images/sds-local-volume-csi/internal/const.go @@ -25,6 +25,7 @@ const ( TopologyKey = "topology.sds-local-volume-csi/node" SubPath = "subPath" VGNameKey = "vgname" + ThinPoolNameKey = "thinPoolName" LLMTypeThin = "Thin" LLMTypeThick = "Thick" LLVStatusCreated = "Created" diff --git a/images/sds-local-volume-csi/pkg/utils/volume.go b/images/sds-local-volume-csi/pkg/utils/volume.go index b3a6f440..d4747ece 100644 --- a/images/sds-local-volume-csi/pkg/utils/volume.go +++ b/images/sds-local-volume-csi/pkg/utils/volume.go @@ -19,17 +19,18 @@ package utils import ( "fmt" "os" + "sds-local-volume-csi/internal" "sds-local-volume-csi/pkg/logger" + "strings" mu "k8s.io/mount-utils" utilexec "k8s.io/utils/exec" ) type NodeStoreManager interface { - Mount(source, target string, isBlock bool, fsType string, readonly bool, mntOpts []string) error + Mount(source, target string, isBlock bool, fsType string, readonly bool, mntOpts []string, lvmType, lvmThinPoolName string) error Unmount(target string) error IsNotMountPoint(target string) (bool, error) - IsMountPointCorrect(mountRefs []string, target string) bool ResizeFS(target string) error } @@ -48,20 +49,20 @@ func NewStore(logger *logger.Logger) *Store { } } -func (s *Store) Mount(source, target string, isBlock bool, fsType string, readonly bool, mntOpts []string) error { +func (s *Store) Mount(devSourcePath, target string, isBlock bool, fsType string, readonly bool, mntOpts []string, lvmType, lvmThinPoolName string) error { s.Log.Info(" ----== Node Mount ==---- ") s.Log.Trace("≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈ Mount options ≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈") - s.Log.Trace(fmt.Sprintf("[mount] params source=%s target=%s fs=%s blockMode=%t mountOptions=%v", source, target, fsType, isBlock, mntOpts)) + s.Log.Trace(fmt.Sprintf("[mount] params source=%s target=%s fs=%s blockMode=%t mountOptions=%v", devSourcePath, target, fsType, isBlock, mntOpts)) s.Log.Trace("≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈ Mount options ≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈") - info, err := os.Stat(source) + info, err := os.Stat(devSourcePath) if err != nil { return fmt.Errorf("failed to stat source device: %w", err) } if (info.Mode() & os.ModeDevice) != os.ModeDevice { - return fmt.Errorf("[NewMount] path %s is not a device", source) + return fmt.Errorf("[NewMount] path %s is not a device", devSourcePath) } s.Log.Trace("≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈ MODE SOURCE ≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈") @@ -91,24 +92,29 @@ func (s *Store) Mount(source, target string, isBlock bool, fsType string, readon s.Log.Trace("≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈ isMountPoint ≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈≈") if isMountPoint { - s.Log.Trace(fmt.Sprintf("Try to find mount refs for source %s", source)) - mountRefs, err := s.NodeStorage.GetMountRefs(source) + mapperSourcePath := toMapperPath(devSourcePath) + s.Log.Trace(fmt.Sprintf("Target %s is a mount point. Checking if it is already mounted to source %s or %s", target, devSourcePath, mapperSourcePath)) + + mountedDevicePath, _, err := mu.GetDeviceNameFromMount(s.NodeStorage.Interface, target) if err != nil { - return fmt.Errorf("failed to get mount refs for source %s: %w", source, err) + return fmt.Errorf("failed to find the device mounted at %s: %w", target, err) } - s.Log.Trace(fmt.Sprintf("Found mount refs: %v", mountRefs)) - mountPointCorrect := s.IsMountPointCorrect(mountRefs, target) - if !mountPointCorrect { - return fmt.Errorf("target %s is a mount point and is not mounted to source %s", target, source) + s.Log.Trace(fmt.Sprintf("Found device mounted at %s: %s", target, mountedDevicePath)) + + if mountedDevicePath != devSourcePath && mountedDevicePath != mapperSourcePath { + return fmt.Errorf("target %s is a mount point and is not mounted to source %s or %s", target, devSourcePath, mapperSourcePath) } - s.Log.Trace(fmt.Sprintf("Target %s is a mount point and already mounted to source %s. Skipping FormatAndMount without any checks", target, source)) + s.Log.Trace(fmt.Sprintf("Target %s is a mount point and already mounted to source %s. Skipping FormatAndMount without any checks", target, devSourcePath)) return nil } s.Log.Trace("-----------------== start FormatAndMount ==---------------") - err = s.NodeStorage.FormatAndMount(source, target, fsType, mntOpts) + if lvmType == internal.LLMTypeThin { + s.Log.Trace(fmt.Sprintf("LVM type is Thin. Ckecking free space in thin pool %s", lvmThinPoolName)) + } + err = s.NodeStorage.FormatAndMount(devSourcePath, target, fsType, mntOpts) if err != nil { return fmt.Errorf("failed to FormatAndMount : %w", err) } @@ -129,7 +135,7 @@ func (s *Store) Mount(source, target string, isBlock bool, fsType string, readon } s.Log.Trace("-----------------== stop Create File ==---------------") s.Log.Trace("-----------------== start Mount ==---------------") - err = s.NodeStorage.Mount(source, target, fsType, mntOpts) + err = s.NodeStorage.Mount(devSourcePath, target, fsType, mntOpts) if err != nil { s.Log.Error(err, "block mount error :") return err @@ -184,11 +190,13 @@ func (s *Store) ResizeFS(mountTarget string) error { return nil } -func (s *Store) IsMountPointCorrect(mountRefs []string, target string) bool { - for _, ref := range mountRefs { - if ref == target { - return true - } +func toMapperPath(devPath string) string { + if !strings.HasPrefix(devPath, "/dev/") { + return "" } - return false + + shortPath := strings.TrimPrefix(devPath, "/dev/") + mapperPath := strings.Replace(shortPath, "-", "--", -1) + mapperPath = strings.Replace(mapperPath, "/", "-", -1) + return "/dev/mapper/" + mapperPath } From cf4d2fcf31625ff31f081c16d33fe02e8d6b9e34 Mon Sep 17 00:00:00 2001 From: Aleksandr Zimin Date: Mon, 1 Apr 2024 02:42:01 +0300 Subject: [PATCH 07/21] Some fixes Signed-off-by: Aleksandr Zimin --- images/sds-local-volume-csi/driver/controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/sds-local-volume-csi/driver/controller.go b/images/sds-local-volume-csi/driver/controller.go index 02dabcb9..c3443adb 100644 --- a/images/sds-local-volume-csi/driver/controller.go +++ b/images/sds-local-volume-csi/driver/controller.go @@ -149,7 +149,7 @@ func (d *Driver) CreateVolume(ctx context.Context, request *csi.CreateVolumeRequ } volumeCtx[internal.SubPath] = request.Name - volumeCtx[internal.VGNameKey] = llvSpec.LvmVolumeGroupName + volumeCtx[internal.VGNameKey] = selectedLVG.Spec.ActualVGNameOnTheNode if llvSpec.Type == internal.LLMTypeThin { volumeCtx[internal.ThinPoolNameKey] = llvSpec.Thin.PoolName } else { From b5df71f42e9372ea615d6121be24a0e56938fb0d Mon Sep 17 00:00:00 2001 From: Aleksandr Zimin Date: Mon, 1 Apr 2024 02:52:49 +0300 Subject: [PATCH 08/21] Fix typo Signed-off-by: Aleksandr Zimin --- images/sds-local-volume-csi/driver/controller.go | 6 +++--- images/sds-local-volume-csi/internal/const.go | 4 ++-- images/sds-local-volume-csi/pkg/utils/func.go | 6 +++--- images/sds-local-volume-csi/pkg/utils/volume.go | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/images/sds-local-volume-csi/driver/controller.go b/images/sds-local-volume-csi/driver/controller.go index c3443adb..fce226eb 100644 --- a/images/sds-local-volume-csi/driver/controller.go +++ b/images/sds-local-volume-csi/driver/controller.go @@ -90,7 +90,7 @@ func (d *Driver) CreateVolume(ctx context.Context, request *csi.CreateVolumeRequ preferredNode = selectedNodeName d.log.Info(fmt.Sprintf("Selected node: %s, free space %s ", selectedNodeName, freeSpace.String())) - if LvmType == internal.LLMTypeThick { + if LvmType == internal.LVMTypeThick { if llvSize.Value() > freeSpace.Value() { return nil, status.Errorf(codes.Internal, "requested size: %s is greater than free space: %s", llvSize.String(), freeSpace.String()) } @@ -150,7 +150,7 @@ func (d *Driver) CreateVolume(ctx context.Context, request *csi.CreateVolumeRequ volumeCtx[internal.SubPath] = request.Name volumeCtx[internal.VGNameKey] = selectedLVG.Spec.ActualVGNameOnTheNode - if llvSpec.Type == internal.LLMTypeThin { + if llvSpec.Type == internal.LVMTypeThin { volumeCtx[internal.ThinPoolNameKey] = llvSpec.Thin.PoolName } else { volumeCtx[internal.ThinPoolNameKey] = "" @@ -306,7 +306,7 @@ func (d *Driver) ControllerExpandVolume(ctx context.Context, request *csi.Contro return nil, status.Errorf(codes.Internal, "error getting LVMVolumeGroup: %v", err) } - if llv.Spec.Type == internal.LLMTypeThick { + if llv.Spec.Type == internal.LVMTypeThick { lvgFreeSpace, err := utils.GetLVMVolumeGroupFreeSpace(*lvg) if err != nil { return nil, status.Errorf(codes.Internal, "error getting LVMVolumeGroupCapacity: %v", err) diff --git a/images/sds-local-volume-csi/internal/const.go b/images/sds-local-volume-csi/internal/const.go index 9bd59a5e..527c6094 100644 --- a/images/sds-local-volume-csi/internal/const.go +++ b/images/sds-local-volume-csi/internal/const.go @@ -26,8 +26,8 @@ const ( SubPath = "subPath" VGNameKey = "vgname" ThinPoolNameKey = "thinPoolName" - LLMTypeThin = "Thin" - LLMTypeThick = "Thick" + LVMTypeThin = "Thin" + LVMTypeThick = "Thick" LLVStatusCreated = "Created" BindingModeWFFC = "WaitForFirstConsumer" BindingModeI = "Immediate" diff --git a/images/sds-local-volume-csi/pkg/utils/func.go b/images/sds-local-volume-csi/pkg/utils/func.go index b6dce0b2..0aa29c25 100644 --- a/images/sds-local-volume-csi/pkg/utils/func.go +++ b/images/sds-local-volume-csi/pkg/utils/func.go @@ -177,12 +177,12 @@ func GetNodeWithMaxFreeSpace(log *logger.Logger, lvgs []v1alpha1.LvmVolumeGroup, for _, lvg := range lvgs { switch lvmType { - case internal.LLMTypeThick: + case internal.LVMTypeThick: freeSpace, err = GetLVMVolumeGroupFreeSpace(lvg) if err != nil { return "", freeSpace, fmt.Errorf("get free space for lvg %+v: %w", lvg, err) } - case internal.LLMTypeThin: + case internal.LVMTypeThin: thinPoolName, ok := storageClassLVGParametersMap[lvg.Name] if !ok { return "", freeSpace, fmt.Errorf("thin pool name for lvg %s not found in storage class parameters: %+v", lvg.Name, storageClassLVGParametersMap) @@ -384,7 +384,7 @@ func GetLVGList(ctx context.Context, kc client.Client) (*v1alpha1.LvmVolumeGroup func GetLLVSpec(log *logger.Logger, lvName string, selectedLVG v1alpha1.LvmVolumeGroup, storageClassLVGParametersMap map[string]string, nodeName, lvmType string, llvSize resource.Quantity) v1alpha1.LVMLogicalVolumeSpec { var llvThin *v1alpha1.ThinLogicalVolumeSpec - if lvmType == internal.LLMTypeThin { + if lvmType == internal.LVMTypeThin { llvThin = &v1alpha1.ThinLogicalVolumeSpec{} llvThin.PoolName = storageClassLVGParametersMap[selectedLVG.Name] log.Info(fmt.Sprintf("[GetLLVSpec] Thin pool name: %s", llvThin.PoolName)) diff --git a/images/sds-local-volume-csi/pkg/utils/volume.go b/images/sds-local-volume-csi/pkg/utils/volume.go index d4747ece..3444e87a 100644 --- a/images/sds-local-volume-csi/pkg/utils/volume.go +++ b/images/sds-local-volume-csi/pkg/utils/volume.go @@ -111,7 +111,7 @@ func (s *Store) Mount(devSourcePath, target string, isBlock bool, fsType string, s.Log.Trace("-----------------== start FormatAndMount ==---------------") - if lvmType == internal.LLMTypeThin { + if lvmType == internal.LVMTypeThin { s.Log.Trace(fmt.Sprintf("LVM type is Thin. Ckecking free space in thin pool %s", lvmThinPoolName)) } err = s.NodeStorage.FormatAndMount(devSourcePath, target, fsType, mntOpts) From d3df202c5fc7627cd3d00220e2e5056b31eb817c Mon Sep 17 00:00:00 2001 From: Aleksandr Zimin Date: Mon, 1 Apr 2024 19:47:25 +0300 Subject: [PATCH 09/21] fix assignment to entry in nil map Signed-off-by: Aleksandr Zimin --- .../controller/local_storage_class_watcher.go | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/images/sds-local-volume-controller/pkg/controller/local_storage_class_watcher.go b/images/sds-local-volume-controller/pkg/controller/local_storage_class_watcher.go index 9ed83d59..7aa529be 100644 --- a/images/sds-local-volume-controller/pkg/controller/local_storage_class_watcher.go +++ b/images/sds-local-volume-controller/pkg/controller/local_storage_class_watcher.go @@ -20,6 +20,14 @@ import ( "context" "errors" "fmt" + "reflect" + v1alpha1 "sds-local-volume-controller/api/v1alpha1" + "sds-local-volume-controller/pkg/config" + "sds-local-volume-controller/pkg/logger" + "sds-local-volume-controller/pkg/monitoring" + "strings" + "time" + corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/storage/v1" errors2 "k8s.io/apimachinery/pkg/api/errors" @@ -27,11 +35,6 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/workqueue" "k8s.io/utils/strings/slices" - "reflect" - v1alpha1 "sds-local-volume-controller/api/v1alpha1" - "sds-local-volume-controller/pkg/config" - "sds-local-volume-controller/pkg/logger" - "sds-local-volume-controller/pkg/monitoring" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" @@ -39,8 +42,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" "sigs.k8s.io/yaml" - "strings" - "time" "sigs.k8s.io/controller-runtime/pkg/manager" ) @@ -443,13 +444,19 @@ func reconcileLSCUpdateFunc( } func patchSCByLSC(sc *v1.StorageClass, lsc *v1alpha1.LocalStorageClass) *v1.StorageClass { - lscDefault := "false" + if lsc.Spec.IsDefault { - lscDefault = "true" + if sc.Annotations == nil { + sc.Annotations = make(map[string]string) + } + sc.Annotations[DefaultStorageClassAnnotationKey] = "true" + } else { + _, isDefault := sc.Annotations[DefaultStorageClassAnnotationKey] + if isDefault { + delete(sc.Annotations, DefaultStorageClassAnnotationKey) + } } - sc.Annotations[DefaultStorageClassAnnotationKey] = lscDefault - return sc } From bc4618bac135bfe60805c7e796285703868b0948 Mon Sep 17 00:00:00 2001 From: Viktor Kramarenko Date: Mon, 1 Apr 2024 20:02:44 +0300 Subject: [PATCH 10/21] working version Signed-off-by: Viktor Kramarenko --- .../pkg/cache/cache.go | 157 +++++++++--- .../pkg/controller/lvg_watcher_cache.go | 58 ++--- .../pkg/controller/pvc_watcher_cache.go | 236 ++++++++++-------- .../pkg/scheduler/filter.go | 179 +++++++++---- .../pkg/scheduler/prioritize.go | 32 ++- .../pkg/scheduler/route.go | 18 +- 6 files changed, 431 insertions(+), 249 deletions(-) diff --git a/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go b/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go index 71a58f01..ee309b4f 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go +++ b/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go @@ -4,13 +4,19 @@ import ( "fmt" v1 "k8s.io/api/core/v1" "sds-local-volume-scheduler-extender/api/v1alpha1" + "sds-local-volume-scheduler-extender/pkg/logger" ) -const pvcPerLVGCount = 150 +const ( + pvcPerLVGCount = 150 + SelectedNodeAnnotation = "volume.kubernetes.io/selected-node" +) type Cache struct { - lvgs map[string]*lvgCache - pvcLVG map[string]string + lvgs map[string]*lvgCache + pvcQueue map[string]*v1.PersistentVolumeClaim + pvcLVGs map[string][]string + nodeLVGs map[string][]string } type lvgCache struct { @@ -25,8 +31,10 @@ type pvcCache struct { func NewCache(size int) *Cache { return &Cache{ - lvgs: make(map[string]*lvgCache, size), - pvcLVG: make(map[string]string, size), + lvgs: make(map[string]*lvgCache, size), + pvcQueue: make(map[string]*v1.PersistentVolumeClaim, size), + pvcLVGs: make(map[string][]string, size), + nodeLVGs: make(map[string][]string, size), } } @@ -40,6 +48,10 @@ func (c *Cache) AddLVG(lvg *v1alpha1.LvmVolumeGroup) error { pvcs: make(map[string]*pvcCache, pvcPerLVGCount), } + for _, node := range lvg.Status.Nodes { + c.nodeLVGs[node.Name] = append(c.nodeLVGs[node.Name], lvg.Name) + } + return nil } @@ -60,13 +72,17 @@ func (c *Cache) TryGetLVG(name string) *v1alpha1.LvmVolumeGroup { return c.lvgs[name].lvg } -func (c *Cache) GetLVGNames() []string { - names := make([]string, 0, len(c.lvgs)) - for lvgName := range c.lvgs { - names = append(names, lvgName) +func (c *Cache) GetLVGNamesByNodeName(nodeName string) []string { + return c.nodeLVGs[nodeName] +} + +func (c *Cache) GetAllLVG() map[string]*v1alpha1.LvmVolumeGroup { + lvgs := make(map[string]*v1alpha1.LvmVolumeGroup, len(c.lvgs)) + for _, lvgCh := range c.lvgs { + lvgs[lvgCh.lvg.Name] = lvgCh.lvg } - return names + return lvgs } func (c *Cache) GetLVGReservedSpace(lvgName string) int64 { @@ -84,40 +100,70 @@ func (c *Cache) DeleteLVG(lvgName string) { delete(c.lvgs, lvgName) } -func (c *Cache) AddPVC(lvgName string, pvc *v1.PersistentVolumeClaim, pvcNodeName string) error { - if c.lvgs[lvgName].pvcs[pvc.Name] != nil { +func (c *Cache) AddPVCToFilterQueue(pvc *v1.PersistentVolumeClaim) { + pvcKey := configurePVCKey(pvc) + c.pvcQueue[pvcKey] = pvc +} + +func (c *Cache) AddUnboundedPVCToLVG(lvgName string, pvc *v1.PersistentVolumeClaim) error { + if pvc.Annotations[SelectedNodeAnnotation] != "" { + return fmt.Errorf("PVC %s is expected to not have a selected node, but got one: %s", pvc.Name, pvc.Annotations[SelectedNodeAnnotation]) + } + + pvcKey := configurePVCKey(pvc) + if c.lvgs[lvgName].pvcs[pvcKey] != nil { return fmt.Errorf("PVC %s already exist in the cache", pvc.Name) } - c.lvgs[lvgName].pvcs[pvc.Name] = &pvcCache{pvc: pvc, nodeName: pvcNodeName} - c.pvcLVG[pvc.Name] = lvgName + c.lvgs[lvgName].pvcs[pvcKey] = &pvcCache{pvc: pvc, nodeName: ""} + c.pvcLVGs[pvcKey] = append(c.pvcLVGs[pvcKey], lvgName) + return nil } -func (c *Cache) UpdatePVC(pvc *v1.PersistentVolumeClaim, pvcNodeName string) error { - lvgName := c.pvcLVG[pvc.Name] - if c.lvgs[lvgName].pvcs[pvc.Name] == nil { +func (c *Cache) UpdatePVC(lvgName string, pvc *v1.PersistentVolumeClaim) error { + pvcKey := configurePVCKey(pvc) + if c.lvgs[lvgName].pvcs[pvcKey] == nil { return fmt.Errorf("PVC %s not found", pvc.Name) } - c.lvgs[lvgName].pvcs[pvc.Name].pvc = pvc - c.lvgs[lvgName].pvcs[pvc.Name].nodeName = pvcNodeName + c.lvgs[lvgName].pvcs[pvcKey].pvc = pvc + c.lvgs[lvgName].pvcs[pvcKey].nodeName = pvc.Annotations[SelectedNodeAnnotation] + return nil } -func (c *Cache) TryGetPVC(name string) *v1.PersistentVolumeClaim { - lvgName := c.pvcLVG[name] +func (c *Cache) TryGetPVC(lvgName string, pvc *v1.PersistentVolumeClaim) *v1.PersistentVolumeClaim { + pvcKey := configurePVCKey(pvc) if c.lvgs[lvgName] == nil { return nil } - return c.lvgs[lvgName].pvcs[name].pvc + return c.lvgs[lvgName].pvcs[pvcKey].pvc } -//func (c *Cache) GetCorrespondingLVGNameByPVC(pvcName string) string { -// return c.pvcLVG[pvcName] -//} +func (c *Cache) CheckPVCInQueue(pvc *v1.PersistentVolumeClaim) bool { + pvcKey := configurePVCKey(pvc) + + if _, exist := c.pvcQueue[pvcKey]; !exist { + return false + } + + return true +} + +func (c *Cache) GetPVCsFromQueue(namespace string) map[string]*v1.PersistentVolumeClaim { + result := make(map[string]*v1.PersistentVolumeClaim, len(c.pvcQueue)) + + for _, pvc := range c.pvcQueue { + if pvc.Namespace == namespace { + result[pvc.Name] = pvc + } + } + + return result +} func (c *Cache) GetAllPVCByLVG(lvgName string) []*v1.PersistentVolumeClaim { pvcsCache := c.lvgs[lvgName] @@ -130,22 +176,61 @@ func (c *Cache) GetAllPVCByLVG(lvgName string) []*v1.PersistentVolumeClaim { return result } -func (c *Cache) GetPVCNodeName(pvcName string) string { - lvgName := c.pvcLVG[pvcName] - return c.lvgs[lvgName].pvcs[pvcName].nodeName +func (c *Cache) GetPVCNodeName(lvgName string, pvc *v1.PersistentVolumeClaim) string { + pvcKey := configurePVCKey(pvc) + return c.lvgs[lvgName].pvcs[pvcKey].nodeName } -func (c *Cache) GetAllPVCNames() []string { - result := make([]string, 0, len(c.pvcLVG)) +func (c *Cache) GetLVGNamesForPVC(pvc *v1.PersistentVolumeClaim) []string { + pvcKey := configurePVCKey(pvc) + return c.pvcLVGs[pvcKey] +} - for pvcName := range c.pvcLVG { - result = append(result, pvcName) +func (c *Cache) RemoveBoundedPVCSpaceReservation(lvgName string, pvc *v1.PersistentVolumeClaim) error { + pvcKey := configurePVCKey(pvc) + pvcCh := c.lvgs[lvgName].pvcs[pvcKey] + if pvcCh.nodeName == "" { + return fmt.Errorf("no node selected for PVC %s", pvc.Name) } - return result + delete(c.lvgs[lvgName].pvcs, pvcKey) + delete(c.pvcLVGs, pvcKey) + delete(c.pvcQueue, pvcKey) + + return nil +} + +func (c *Cache) RemoveUnboundedPVCSpaceReservation(log logger.Logger, pvc *v1.PersistentVolumeClaim) error { + pvcKey := configurePVCKey(pvc) + + for _, lvgCh := range c.lvgs { + pvcCh, exist := lvgCh.pvcs[pvcKey] + if exist { + if pvcCh.nodeName == "" { + delete(lvgCh.pvcs, pvcKey) + delete(c.pvcQueue, pvcKey) + log.Debug(fmt.Sprintf("[RemoveUnboundedPVCSpaceReservation] removed unbound cache PVC %s from LVG %s", pvc.Name, lvgCh.lvg.Name)) + continue + } + + log.Debug(fmt.Sprintf("[RemoveUnboundedPVCSpaceReservation] PVC %s has bounded to node %s. It should not be revomed from LVG %s", pvc.Name, pvcCh.nodeName, lvgCh.lvg.Name)) + } + } + + return nil +} + +func (c *Cache) RemovePVCSpaceReservationForced(pvc *v1.PersistentVolumeClaim) { + pvcKey := configurePVCKey(pvc) + + for _, lvgName := range c.pvcLVGs[pvcKey] { + delete(c.lvgs[lvgName].pvcs, pvcKey) + } + + delete(c.pvcLVGs, pvcKey) + delete(c.pvcQueue, pvcKey) } -func (c *Cache) RemovePVCSpaceReservation(pvcName string) { - lvgName := c.pvcLVG[pvcName] - delete(c.lvgs[lvgName].pvcs, pvcName) +func configurePVCKey(pvc *v1.PersistentVolumeClaim) string { + return fmt.Sprintf("%s/%s", pvc.Namespace, pvc.Name) } diff --git a/images/sds-local-volume-scheduler-extender/pkg/controller/lvg_watcher_cache.go b/images/sds-local-volume-scheduler-extender/pkg/controller/lvg_watcher_cache.go index 0fef0ac2..7329511f 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/controller/lvg_watcher_cache.go +++ b/images/sds-local-volume-scheduler-extender/pkg/controller/lvg_watcher_cache.go @@ -33,7 +33,8 @@ func RunLVGWatcherCacheController( c, err := controller.New(LVGWatcherCacheCtrlName, mgr, controller.Options{ Reconciler: reconcile.Func(func(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { return reconcile.Result{}, nil - })}) + }), + }) if err != nil { log.Error(err, "[RunCacheWatcherController] unable to create a controller") return nil, err @@ -77,37 +78,16 @@ func RunLVGWatcherCacheController( log.Info(fmt.Sprintf("[RunLVGWatcherCacheController] cache was added for the LVMVolumeGroup %s", lvg.Name)) } - //pvcs := &v1.PersistentVolumeClaimList{} - //err := cl.List(ctx, pvcs) - //if err != nil { - // log.Error(err, "[RunLVGWatcherCacheController] unable to list all PVCs") - // // TODO: requeue - // return - //} - // - //scsList := &v12.StorageClassList{} - //err = cl.List(ctx, scsList) - //if err != nil { - // log.Error(err, "[RunLVGWatcherCacheController] unable to list all StorageClasses") - // // TODO: requeue - // return - //} - //scs := make(map[string]v12.StorageClass, len(scsList.Items)) - //for _, sc := range scsList.Items { - // scs[sc.Name] = sc - //} - // - //lvgsBySC, err := scheduler.GetSortedLVGsFromStorageClasses(scs) - //if err != nil { - // log.Error(err, "[RunLVGWatcherCacheController] unable to sort LVGs by StorageClasses") - // // TODO: requeue - // return - //} - pvcs := cache.GetAllPVCByLVG(lvg.Name) for _, pvc := range pvcs { if pvc.Status.Phase == v1.ClaimBound { - cache.RemovePVCSpaceReservation(pvc.Name) + err = cache.RemoveBoundedPVCSpaceReservation(lvg.Name, pvc) + if err != nil { + log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to remove PVC %s from the cache for the LVG %s", pvc.Name, lvg.Name)) + continue + } + + log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] PVC %s was removed from the LVG %s in the cache", pvc.Name, lvg.Name)) } } @@ -150,16 +130,26 @@ func RunLVGWatcherCacheController( err = cache.UpdateLVG(newLvg) if err != nil { log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to update the LVMVolumeGroup %s cache", newLvg.Name)) + return } - pvcs := cache.GetAllPVCByLVG(newLvg.Name) - for _, pvc := range pvcs { + cachedPvcs := cache.GetAllPVCByLVG(newLvg.Name) + for _, pvc := range cachedPvcs { + log.Trace(fmt.Sprintf("[RunLVGWatcherCacheController] PVC %s from the cache belongs to LVG %s", pvc.Name, newLvg.Name)) + log.Trace(fmt.Sprintf("[RunLVGWatcherCacheController] PVC %s has status phase %s", pvc.Name, pvc.Status.Phase)) if pvc.Status.Phase == v1.ClaimBound { - cache.RemovePVCSpaceReservation(pvc.Name) + log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] PVC %s from the cache has status phase Bound. It will be removed from the reserved space of LVG %s", pvc.Name, newLvg.Name)) + err = cache.RemoveBoundedPVCSpaceReservation(newLvg.Name, pvc) + if err != nil { + log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to remove PVC %s from the cache for the LVG %s", pvc.Name, newLvg.Name)) + continue + } + + log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] PVC %s was removed from the LVG %s in the cache", pvc.Name, newLvg.Name)) } } - log.Info(fmt.Sprintf("[RunLVGWatcherCacheController] updated LVMVolumeGroup %s cache size", newLvg.Name)) + log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] reconciled LVMVolumeGroup %s cache", newLvg.Name)) }, DeleteFunc: func(ctx context.Context, e event.DeleteEvent, q workqueue.RateLimitingInterface) { log.Info(fmt.Sprintf("[RunCacheWatcherController] DeleteFunc starts the cache reconciliation for the LVMVolumeGroup %s", e.Object.GetName())) @@ -170,7 +160,7 @@ func RunLVGWatcherCacheController( return } cache.DeleteLVG(lvg.Name) - log.Info(fmt.Sprintf("[RunLVGWatcherCacheController] LVMVolumeGroup %s was deleted from the cache", lvg.Name)) + log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] LVMVolumeGroup %s was deleted from the cache", lvg.Name)) }, }) if err != nil { diff --git a/images/sds-local-volume-scheduler-extender/pkg/controller/pvc_watcher_cache.go b/images/sds-local-volume-scheduler-extender/pkg/controller/pvc_watcher_cache.go index 4e25b410..9781c8f5 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/controller/pvc_watcher_cache.go +++ b/images/sds-local-volume-scheduler-extender/pkg/controller/pvc_watcher_cache.go @@ -5,19 +5,15 @@ import ( "errors" "fmt" v1 "k8s.io/api/core/v1" - v12 "k8s.io/api/storage/v1" cache2 "k8s.io/client-go/tools/cache" - "sds-local-volume-scheduler-extender/api/v1alpha1" + "k8s.io/utils/strings/slices" "sds-local-volume-scheduler-extender/pkg/cache" "sds-local-volume-scheduler-extender/pkg/logger" - "sds-local-volume-scheduler-extender/pkg/scheduler" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" ) const ( PVCWatcherCacheCtrlName = "pvc-watcher-cache-controller" - selectedNodeAnnotation = "volume.kubernetes.io/selected-node" ) func RunPVCWatcherCacheController( @@ -28,15 +24,15 @@ func RunPVCWatcherCacheController( ) error { log.Info("[RunPVCWatcherCacheController] starts the work WITH EVENTS") - cl := mgr.GetClient() inf, err := mgr.GetCache().GetInformer(ctx, &v1.PersistentVolumeClaim{}) if err != nil { log.Error(err, "[RunPVCWatcherCacheController] unable to get the informer") return err } - inf.AddEventHandler(cache2.ResourceEventHandlerFuncs{ + _, err = inf.AddEventHandler(cache2.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { + log.Info("[RunPVCWatcherCacheController] Add Func reconciliation starts") pvc, ok := obj.(*v1.PersistentVolumeClaim) if !ok { err = errors.New("unable to cast event object to a given type") @@ -44,131 +40,163 @@ func RunPVCWatcherCacheController( } log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] AddFunc starts the reconciliation for the PVC %s", pvc.Name)) - if shouldReconcilvePVC(pvc) { - cachedPvc := schedulerCache.TryGetPVC(pvc.Name) - if cachedPvc != nil { - if pvc.Status.Phase == v1.ClaimBound { - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s in a Bound state. It will be removed from the cache")) - //schedulerCache.RemovePVCSpaceReservation(pvc.Name) - log.Info(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s was removed from the cache")) - return - } + switch shouldAddPVCToCache(schedulerCache, pvc) { + case true: + // Добавляем в queue, иначе фильтр не сможет получить ее из кеша + schedulerCache.AddPVCToFilterQueue(pvc) + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s was added to the cache", pvc.Name)) + case false: + // Update Func (если рухнули на апдейте) + selectedNode, wasSelected := pvc.Annotations[cache.SelectedNodeAnnotation] + if !wasSelected { + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s should not be reconciled by Add Func due it has not selected node annotation", pvc.Name)) + return + } + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s has node annotation, it will be reconciled in Add func", pvc.Name)) - err := schedulerCache.UpdatePVC(pvc, pvc.Annotations[selectedNodeAnnotation]) - if err != nil { - log.Error(err, fmt.Sprintf("[RunPVCWatcherCacheController] unable to update PVC %s in the cache", pvc.Name)) - } - } else { - lvg, err := findLVGByPVC(ctx, cl, pvc) - if err != nil { - log.Error(err, "[RunPVCWatcherCacheController] an error occurs the founding a LVMVolumeGroup") - // TODO: requeue or something - return - } + //cachedPvc := schedulerCache.TryGetPVC(pvc) + //if cachedPvc == nil { + // log.Error(fmt.Errorf("PVC %s was not found in the cache", pvc.Name), fmt.Sprintf("[RunPVCWatcherCacheController] unable to get PVC %s from the cache", pvc.Name)) + // return + //} + + lvgsOnTheNode := schedulerCache.GetLVGNamesByNodeName(selectedNode) + lvgsForPVC := schedulerCache.GetLVGNamesForPVC(pvc) - err = schedulerCache.AddPVC(lvg.Name, pvc, pvc.Annotations[selectedNodeAnnotation]) - if err != nil { - log.Error(err, "[RunPVCWatcherCacheController] unable to add PVC to the cache") - // TODO: requeue or something - return + var lvgName string + for _, pvcLvg := range lvgsForPVC { + if slices.Contains(lvgsOnTheNode, pvcLvg) { + lvgName = pvcLvg } } - } + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s has status phase: %s", pvc.Name, pvc.Status.Phase)) + err = schedulerCache.UpdatePVC(lvgName, pvc) + if err != nil { + log.Error(err, fmt.Sprintf("[RunPVCWatcherCacheController] unable to update PVC %s in the cache", pvc.Name)) + return + } + + // У PVC выбралась нода, но она еще не в баунд (в кеше PVC без ноды на лвгхах) + err = schedulerCache.RemoveUnboundedPVCSpaceReservation(log, pvc) + if err != nil { + log.Error(err, fmt.Sprintf("[RunPVCWatcherCacheController] unable to remove space reservation in the cache for unbounded PVC %s", pvc.Name)) + } + } }, UpdateFunc: func(oldObj, newObj interface{}) { + log.Info("[RunPVCWatcherCacheController] Update Func reconciliation starts") pvc, ok := newObj.(*v1.PersistentVolumeClaim) if !ok { err = errors.New("unable to cast event object to a given type") log.Error(err, "[RunPVCWatcherCacheController] an error occurred while handling create event") } - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] AddFunc starts the reconciliation for the PVC %s", pvc.Name)) + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] UpdateFunc starts the reconciliation for the PVC %s", pvc.Name)) - if shouldReconcilvePVC(pvc) { - cachedPvc := schedulerCache.TryGetPVC(pvc.Name) - if cachedPvc != nil { - if pvc.Status.Phase == v1.ClaimBound { - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s in a Bound state. It will be removed from the cache")) - //schedulerCache.RemovePVCSpaceReservation(pvc.Name) - log.Info(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s was removed from the cache")) - return - } + selectedNode, wasSelected := pvc.Annotations[cache.SelectedNodeAnnotation] + if !wasSelected { + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s should not be reconciled by Add Func due it has not selected node annotation", pvc.Name)) + return + } + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s has node annotation, it will be reconciled in Update func", pvc.Name)) - err := schedulerCache.UpdatePVC(pvc, pvc.Annotations[selectedNodeAnnotation]) - if err != nil { - log.Error(err, fmt.Sprintf("[RunPVCWatcherCacheController] unable to update PVC %s in the cache", pvc.Name)) - } - } else { - lvg, err := findLVGByPVC(ctx, cl, pvc) - if err != nil { - log.Error(err, "[RunPVCWatcherCacheController] an error occurs the founding a LVMVolumeGroup") - // TODO: requeue or something - return - } + lvgsOnTheNode := schedulerCache.GetLVGNamesByNodeName(selectedNode) + lvgsForPVC := schedulerCache.GetLVGNamesForPVC(pvc) - err = schedulerCache.AddPVC(lvg.Name, pvc, pvc.Annotations[selectedNodeAnnotation]) - if err != nil { - log.Error(err, "[RunPVCWatcherCacheController] unable to add PVC to the cache") - // TODO: requeue or something - return - } + var lvgName string + for _, pvcLvg := range lvgsForPVC { + if slices.Contains(lvgsOnTheNode, pvcLvg) { + lvgName = pvcLvg } } + log.Trace(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s uses LVG %s on node %s", pvc.Name, lvgName, selectedNode)) + log.Trace(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s has status phase: %s", pvc.Name, pvc.Status.Phase)) + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] updates cache PVC %s in LVG %s", pvc.Name, lvgName)) + err = schedulerCache.UpdatePVC(lvgName, pvc) + if err != nil { + log.Error(err, fmt.Sprintf("[RunPVCWatcherCacheController] unable to update PVC %s in the cache", pvc.Name)) + return + } + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] successfully updated cache PVC %s in LVG %s", pvc.Name, lvgName)) + + // У PVC выбралась нода, но она еще не в баунд (в кеше PVC без ноды на лвгхах) + err = schedulerCache.RemoveUnboundedPVCSpaceReservation(log, pvc) + if err != nil { + log.Error(err, fmt.Sprintf("[RunPVCWatcherCacheController] unable to remove space reservation in the cache for unbounded PVC %s", pvc.Name)) + } }, DeleteFunc: func(obj interface{}) { - log.Info("HELLO FROM DELETEFUNC") + log.Info("[RunPVCWatcherCacheController] Delete Func reconciliation starts") + pvc, ok := obj.(*v1.PersistentVolumeClaim) + if !ok { + err = errors.New("unable to cast event object to a given type") + log.Error(err, "[RunPVCWatcherCacheController] an error occurred while handling create event") + } + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] DeleteFunc starts the reconciliation for the PVC %s", pvc.Name)) + + schedulerCache.RemovePVCSpaceReservationForced(pvc) + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s was force removed from the cache", pvc.Name)) }, }) + if err != nil { + log.Error(err, "[RunPVCWatcherCacheController] unable to add event handler to the informer") + } return nil } -func shouldReconcilvePVC(pvc *v1.PersistentVolumeClaim) bool { - _, selected := pvc.Annotations[selectedNodeAnnotation] - if pvc.Status.Phase != v1.ClaimBound && selected { +func shouldAddPVCToCache(schedulerCache *cache.Cache, pvc *v1.PersistentVolumeClaim) bool { + //_, selected := pvc.Annotations[cache.SelectedNodeAnnotation] + if pvc.Status.Phase != v1.ClaimBound { return true } - return false -} - -func findLVGByPVC(ctx context.Context, cl client.Client, pvc *v1.PersistentVolumeClaim) (*v1alpha1.LvmVolumeGroup, error) { - sc := &v12.StorageClass{} - // TODO: Будет ли проставлен storage class для PVC, если не будет указан явно (иначе зачем тут поинтер?) - err := cl.Get(ctx, client.ObjectKey{ - Name: *pvc.Spec.StorageClassName, - }, sc) - if err != nil { - return nil, fmt.Errorf("[findLVGByPVC] unable to get a storage class %s", *pvc.Spec.StorageClassName) - } - - lvgsFromSC, err := scheduler.ExtractLVGsFromSC(*sc) - - lvgList := &v1alpha1.LvmVolumeGroupList{} - err = cl.List(ctx, lvgList) - if err != nil { - return nil, fmt.Errorf("[findLVGByPVC] unable to list LVMVolumeGroups") - } - - lvgs := make(map[string]v1alpha1.LvmVolumeGroup, len(lvgList.Items)) - for _, lvg := range lvgList.Items { - lvgs[lvg.Name] = lvg - } - - for _, lvg := range lvgsFromSC { - kubeLVG, exist := lvgs[lvg.Name] - if !exist { - return nil, fmt.Errorf("unable to found the LVMVolumeGroup %s for storage class %s", lvg.Name, sc.Name) - } - - if kubeLVG.Status.Nodes == nil || len(kubeLVG.Status.Nodes) == 0 { - return nil, fmt.Errorf("no nodes specified for the LVMVolumeGroup %s for storage class %s", lvg.Name, sc.Name) - } - - if kubeLVG.Status.Nodes[0].Name == pvc.Annotations[selectedNodeAnnotation] { - return &kubeLVG, nil - } + exist := schedulerCache.CheckPVCInQueue(pvc) + if !exist { + return true } - return nil, nil + return false } + +// +//func findLVGByPVC(ctx context.Context, cl client.Client, pvc *v1.PersistentVolumeClaim) (*v1alpha1.LvmVolumeGroup, error) { +// sc := &v12.StorageClass{} +// // TODO: Будет ли проставлен storage class для PVC, если не будет указан явно (иначе зачем тут поинтер?) +// err := cl.Get(ctx, client.ObjectKey{ +// Name: *pvc.Spec.StorageClassName, +// }, sc) +// if err != nil { +// return nil, fmt.Errorf("[findLVGByPVC] unable to get a storage class %s", *pvc.Spec.StorageClassName) +// } +// +// lvgsFromSC, err := scheduler.ExtractLVGsFromSC(sc) +// +// lvgList := &v1alpha1.LvmVolumeGroupList{} +// err = cl.List(ctx, lvgList) +// if err != nil { +// return nil, fmt.Errorf("[findLVGByPVC] unable to list LVMVolumeGroups") +// } +// +// lvgs := make(map[string]v1alpha1.LvmVolumeGroup, len(lvgList.Items)) +// for _, lvg := range lvgList.Items { +// lvgs[lvg.Name] = lvg +// } +// +// for _, lvg := range lvgsFromSC { +// kubeLVG, exist := lvgs[lvg.Name] +// if !exist { +// return nil, fmt.Errorf("unable to found the LVMVolumeGroup %s for storage class %s", lvg.Name, sc.Name) +// } +// +// if kubeLVG.Status.Nodes == nil || len(kubeLVG.Status.Nodes) == 0 { +// return nil, fmt.Errorf("no nodes specified for the LVMVolumeGroup %s for storage class %s", lvg.Name, sc.Name) +// } +// +// if kubeLVG.Status.Nodes[0].Name == pvc.Annotations[cache.SelectedNodeAnnotation] { +// return &kubeLVG, nil +// } +// } +// +// return nil, nil +//} diff --git a/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go b/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go index 27d4000b..975b4b45 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go +++ b/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go @@ -21,16 +21,18 @@ import ( "encoding/json" "errors" "fmt" - "net/http" - "sds-local-volume-scheduler-extender/api/v1alpha1" - "sds-local-volume-scheduler-extender/pkg/logger" - "sync" - corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/strings/slices" + "net/http" + "sds-local-volume-scheduler-extender/api/v1alpha1" + "sds-local-volume-scheduler-extender/pkg/cache" + "sds-local-volume-scheduler-extender/pkg/logger" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/yaml" + "sync" + "time" ) const ( @@ -56,12 +58,13 @@ func (s *scheduler) filter(w http.ResponseWriter, r *http.Request) { s.log.Trace(fmt.Sprintf("[filter] a node from request, name :%s", n.Name)) } - pvcs, err := getUsedPVC(s.ctx, s.client, input.Pod) - if err != nil { - s.log.Error(err, "[filter] unable to get PVC from the Pod") + pvcs := getUsedPVC(s.log, s.cache, input.Pod) + if len(pvcs) == 0 { + s.log.Error(fmt.Errorf("no PVC was found for pod %s in namespace %s", input.Pod.Name, input.Pod.Namespace), fmt.Sprintf("[filter] unable to get used PVC for Pod %s", input.Pod.Name)) http.Error(w, "bad request", http.StatusBadRequest) return } + for _, pvc := range pvcs { s.log.Trace(fmt.Sprintf("[filter] used PVC: %s", pvc.Name)) } @@ -81,26 +84,73 @@ func (s *scheduler) filter(w http.ResponseWriter, r *http.Request) { if err != nil { s.log.Error(err, fmt.Sprintf("[filter] unable to extract request size for a pod %s", input.Pod.Name)) http.Error(w, "bad request", http.StatusBadRequest) + return } s.log.Debug("[filter] successfully extracted the pvcRequests size") s.log.Debug("[filter] starts to filter the nodes") - result, err := filterNodes(s.ctx, s.client, s.log, input.Nodes, pvcs, scs, pvcRequests) + filteredNodes, err := filterNodes(s.log, s.cache, input.Nodes, pvcs, scs, pvcRequests) if err != nil { s.log.Error(err, "[filter] unable to filter the nodes") http.Error(w, "bad request", http.StatusBadRequest) + return } s.log.Debug("[filter] successfully filtered the nodes") + s.log.Debug("[filter] starts to populate the cache") + err = populateCache(s.log, filteredNodes.Nodes.Items, input.Pod, s.cache, pvcs, scs) + if err != nil { + s.log.Error(err, "[filter] unable to populate cache") + http.Error(w, "bad request", http.StatusBadRequest) + return + } + s.log.Debug("[filter] successfully populated the cache") + w.Header().Set("content-type", "application/json") - err = json.NewEncoder(w).Encode(result) + err = json.NewEncoder(w).Encode(filteredNodes) if err != nil { s.log.Error(err, "[filter] unable to encode a response") http.Error(w, "internal error", http.StatusInternalServerError) + return } s.log.Debug("[filter] ends the serving") } +func populateCache(log logger.Logger, nodes []corev1.Node, pod *corev1.Pod, schedulerCache *cache.Cache, pvcs map[string]*corev1.PersistentVolumeClaim, scs map[string]*v1.StorageClass) error { + for _, node := range nodes { + for _, volume := range pod.Spec.Volumes { + if volume.PersistentVolumeClaim != nil { + lvgNamesForTheNode := schedulerCache.GetLVGNamesByNodeName(node.Name) + log.Trace(fmt.Sprintf("[populateCache] LVGs from cache for the node %s: %v", node.Name, lvgNamesForTheNode)) + pvc := pvcs[volume.PersistentVolumeClaim.ClaimName] + sc := scs[*pvc.Spec.StorageClassName] + if sc.Parameters[lvmTypeParamKey] == thick { + log.Debug(fmt.Sprintf("[populateCache] Storage Class %s has device type Thick, so the cache will be populated by PVC space requests", sc.Name)) + lvgsForPVC, err := ExtractLVGsFromSC(sc) + if err != nil { + return err + } + log.Trace(fmt.Sprintf("[populateCache] LVGs for PVC %s: %+v", volume.PersistentVolumeClaim.ClaimName, lvgsForPVC)) + + for _, lvg := range lvgsForPVC { + if slices.Contains(lvgNamesForTheNode, lvg.Name) { + log.Trace(fmt.Sprintf("[populateCache] LVG %s in the cache will be populated with unbounded PVC %s", lvg.Name, volume.PersistentVolumeClaim.ClaimName)) + err = schedulerCache.AddUnboundedPVCToLVG(lvg.Name, pvcs[volume.PersistentVolumeClaim.ClaimName]) + if err != nil { + return err + } + } + } + } else { + log.Debug(fmt.Sprintf("[populateCache] Storage Class %s has device type Thin, so the cache should NOT be populated by PVC space requests", sc.Name)) + } + } + } + } + + return nil +} + type PVCRequest struct { DeviceType string RequestedSize int64 @@ -110,8 +160,8 @@ func extractRequestedSize( ctx context.Context, cl client.Client, log logger.Logger, - pvcs map[string]corev1.PersistentVolumeClaim, - scs map[string]v1.StorageClass, + pvcs map[string]*corev1.PersistentVolumeClaim, + scs map[string]*v1.StorageClass, ) (map[string]PVCRequest, error) { pvs, err := getPersistentVolumes(ctx, cl) if err != nil { @@ -161,12 +211,11 @@ func extractRequestedSize( } func filterNodes( - ctx context.Context, - cl client.Client, log logger.Logger, + schedulerCache *cache.Cache, nodes *corev1.NodeList, - pvcs map[string]corev1.PersistentVolumeClaim, - scs map[string]v1.StorageClass, + pvcs map[string]*corev1.PersistentVolumeClaim, + scs map[string]*v1.StorageClass, pvcRequests map[string]PVCRequest, ) (*ExtenderFilterResult, error) { // Param "pvcRequests" is a total amount of the pvcRequests space (both thick and thin) for Pod (i.e. from every PVC) @@ -176,16 +225,22 @@ func filterNodes( }, nil } - lvgs, err := GetLVMVolumeGroups(ctx, cl) - if err != nil { - return nil, err - } + lvgs := schedulerCache.GetAllLVG() lvgsThickFree, err := getLVGThickFreeSpaces(lvgs) if err != nil { return nil, err } - log.Trace(fmt.Sprintf("[filterNodes] LVGs Thick FreeSpace: %+v", lvgsThickFree)) + log.Trace(fmt.Sprintf("[filterNodes] current LVGs Thick FreeSpace on the node: %+v", lvgsThickFree)) + + for lvgName, freeSpace := range lvgsThickFree { + log.Trace(fmt.Sprintf("[filterNodes] current LVG %s Thick free space %s", lvgName, resource.NewQuantity(freeSpace, resource.BinarySI))) + reservedSize := schedulerCache.GetLVGReservedSpace(lvgName) + log.Trace(fmt.Sprintf("[filterNodes] current LVG %s reserved by pvc from cache space %s", lvgName, resource.NewQuantity(reservedSize, resource.BinarySI))) + lvgsThickFree[lvgName] -= reservedSize + } + log.Trace(fmt.Sprintf("[filterNodes] current LVGs Thick FreeSpace with reserved PVC: %+v", lvgsThickFree)) + lvgsThickFreeMutex := &sync.RWMutex{} scLVGs, err := GetSortedLVGsFromStorageClasses(scs) @@ -323,11 +378,11 @@ func filterNodes( return result, nil } -func getLVGThickFreeSpaces(lvgs map[string]v1alpha1.LvmVolumeGroup) (map[string]int64, error) { +func getLVGThickFreeSpaces(lvgs map[string]*v1alpha1.LvmVolumeGroup) (map[string]int64, error) { result := make(map[string]int64, len(lvgs)) for _, lvg := range lvgs { - free, err := getVGFreeSpace(&lvg) + free, err := getVGFreeSpace(lvg) if err != nil { return nil, err } @@ -348,7 +403,7 @@ func findMatchedThinPool(thinPools []v1alpha1.StatusThinPool, name string) *v1al return nil } -func findMatchedLVG(nodeLVGs []v1alpha1.LvmVolumeGroup, scLVGs LVMVolumeGroups) *LVMVolumeGroup { +func findMatchedLVG(nodeLVGs []*v1alpha1.LvmVolumeGroup, scLVGs LVMVolumeGroups) *LVMVolumeGroup { nodeLVGNames := make(map[string]struct{}, len(nodeLVGs)) for _, lvg := range nodeLVGs { nodeLVGNames[lvg.Name] = struct{}{} @@ -363,8 +418,8 @@ func findMatchedLVG(nodeLVGs []v1alpha1.LvmVolumeGroup, scLVGs LVMVolumeGroups) return nil } -func getCommonNodesByStorageClasses(scs map[string]v1.StorageClass, nodesWithLVGs map[string][]v1alpha1.LvmVolumeGroup) (map[string][]v1alpha1.LvmVolumeGroup, error) { - result := make(map[string][]v1alpha1.LvmVolumeGroup, len(nodesWithLVGs)) +func getCommonNodesByStorageClasses(scs map[string]*v1.StorageClass, nodesWithLVGs map[string][]*v1alpha1.LvmVolumeGroup) (map[string][]*v1alpha1.LvmVolumeGroup, error) { + result := make(map[string][]*v1alpha1.LvmVolumeGroup, len(nodesWithLVGs)) for nodeName, lvgs := range nodesWithLVGs { lvgNames := make(map[string]struct{}, len(lvgs)) @@ -401,8 +456,8 @@ func getCommonNodesByStorageClasses(scs map[string]v1.StorageClass, nodesWithLVG return result, nil } -func RemoveUnusedLVGs(lvgs map[string]v1alpha1.LvmVolumeGroup, scsLVGs map[string]LVMVolumeGroups) map[string]v1alpha1.LvmVolumeGroup { - result := make(map[string]v1alpha1.LvmVolumeGroup, len(lvgs)) +func RemoveUnusedLVGs(lvgs map[string]*v1alpha1.LvmVolumeGroup, scsLVGs map[string]LVMVolumeGroups) map[string]*v1alpha1.LvmVolumeGroup { + result := make(map[string]*v1alpha1.LvmVolumeGroup, len(lvgs)) usedLvgs := make(map[string]struct{}, len(lvgs)) for _, scLvgs := range scsLVGs { @@ -420,7 +475,7 @@ func RemoveUnusedLVGs(lvgs map[string]v1alpha1.LvmVolumeGroup, scsLVGs map[strin return result } -func GetSortedLVGsFromStorageClasses(scs map[string]v1.StorageClass) (map[string]LVMVolumeGroups, error) { +func GetSortedLVGsFromStorageClasses(scs map[string]*v1.StorageClass) (map[string]LVMVolumeGroups, error) { result := make(map[string]LVMVolumeGroups, len(scs)) for _, sc := range scs { @@ -445,7 +500,7 @@ type LVMVolumeGroup struct { } type LVMVolumeGroups []LVMVolumeGroup -func ExtractLVGsFromSC(sc v1.StorageClass) (LVMVolumeGroups, error) { +func ExtractLVGsFromSC(sc *v1.StorageClass) (LVMVolumeGroups, error) { var lvmVolumeGroups LVMVolumeGroups err := yaml.Unmarshal([]byte(sc.Parameters[lvmVolumeGroupsParamKey]), &lvmVolumeGroups) if err != nil { @@ -454,8 +509,8 @@ func ExtractLVGsFromSC(sc v1.StorageClass) (LVMVolumeGroups, error) { return lvmVolumeGroups, nil } -func SortLVGsByNodeName(lvgs map[string]v1alpha1.LvmVolumeGroup) map[string][]v1alpha1.LvmVolumeGroup { - sorted := make(map[string][]v1alpha1.LvmVolumeGroup, len(lvgs)) +func SortLVGsByNodeName(lvgs map[string]*v1alpha1.LvmVolumeGroup) map[string][]*v1alpha1.LvmVolumeGroup { + sorted := make(map[string][]*v1alpha1.LvmVolumeGroup, len(lvgs)) for _, lvg := range lvgs { for _, node := range lvg.Status.Nodes { sorted[node.Name] = append(sorted[node.Name], lvg) @@ -465,6 +520,7 @@ func SortLVGsByNodeName(lvgs map[string]v1alpha1.LvmVolumeGroup) map[string][]v1 return sorted } +// TODO: remove it if no need func GetLVMVolumeGroups(ctx context.Context, cl client.Client) (map[string]v1alpha1.LvmVolumeGroup, error) { lvgl := &v1alpha1.LvmVolumeGroupList{} err := cl.List(ctx, lvgl) @@ -521,7 +577,7 @@ func getPersistentVolumes(ctx context.Context, cl client.Client) (map[string]cor return pvMap, nil } -func getStorageClassesUsedByPVCs(ctx context.Context, cl client.Client, pvcs map[string]corev1.PersistentVolumeClaim) (map[string]v1.StorageClass, error) { +func getStorageClassesUsedByPVCs(ctx context.Context, cl client.Client, pvcs map[string]*corev1.PersistentVolumeClaim) (map[string]*v1.StorageClass, error) { scs := &v1.StorageClassList{} err := cl.List(ctx, scs) if err != nil { @@ -533,7 +589,7 @@ func getStorageClassesUsedByPVCs(ctx context.Context, cl client.Client, pvcs map scMap[sc.Name] = sc } - result := make(map[string]v1.StorageClass, len(pvcs)) + result := make(map[string]*v1.StorageClass, len(pvcs)) for _, pvc := range pvcs { if pvc.Spec.StorageClassName == nil { err = errors.New(fmt.Sprintf("not StorageClass specified for PVC %s", pvc.Name)) @@ -542,32 +598,55 @@ func getStorageClassesUsedByPVCs(ctx context.Context, cl client.Client, pvcs map scName := *pvc.Spec.StorageClassName if sc, match := scMap[scName]; match { - result[sc.Name] = sc + result[sc.Name] = &sc } } return result, nil } -func getUsedPVC(ctx context.Context, cl client.Client, pod *corev1.Pod) (map[string]corev1.PersistentVolumeClaim, error) { - usedPvc := make(map[string]corev1.PersistentVolumeClaim, len(pod.Spec.Volumes)) +func getUsedPVC(log logger.Logger, schedulerCache *cache.Cache, pod *corev1.Pod) map[string]*corev1.PersistentVolumeClaim { + usedPvc := make(map[string]*corev1.PersistentVolumeClaim, len(pod.Spec.Volumes)) + //TODO: может сделать так, чтобы в случае, если мы не можем получить из кеша, идти в куб? или будем честно "падать"? - pvcs := &corev1.PersistentVolumeClaimList{} - err := cl.List(ctx, pvcs) - if err != nil { - return nil, err - } + for { + pvcMap := schedulerCache.GetPVCsFromQueue(pod.Namespace) + for _, volume := range pod.Spec.Volumes { + if volume.PersistentVolumeClaim != nil { + usedPvc[volume.PersistentVolumeClaim.ClaimName] = pvcMap[volume.PersistentVolumeClaim.ClaimName] + } + } - pvcMap := make(map[string]corev1.PersistentVolumeClaim, len(pvcs.Items)) - for _, pvc := range pvcs.Items { - pvcMap[pvc.Name] = pvc - } + filled := false + if len(pvcMap) > 0 { + filled = true + for _, pvc := range pvcMap { + log.Trace(fmt.Sprintf("[getUsedPVC] PVC %s from the cache of namespace %s", pvc.Name, pod.Namespace)) + } - for _, volume := range pod.Spec.Volumes { - if volume.PersistentVolumeClaim != nil { - usedPvc[volume.PersistentVolumeClaim.ClaimName] = pvcMap[volume.PersistentVolumeClaim.ClaimName] + for _, volume := range pod.Spec.Volumes { + if volume.PersistentVolumeClaim != nil { + if _, added := usedPvc[volume.PersistentVolumeClaim.ClaimName]; !added { + filled = false + log.Warning(fmt.Sprintf("[getUsedPVC] ClaimName %s was not found in the cache for Pod %s", volume.PersistentVolumeClaim.ClaimName, pod.Name)) + break + } + } + } + } + + if !filled { + log.Warning(fmt.Sprintf("[getUsedPVC] some PVCs were not found in the cache for Pod %s. Retry to find them again.", pod.Name)) + time.Sleep(100 * time.Millisecond) + continue } + + if filled { + log.Debug(fmt.Sprintf("[getUsedPVC] Every claim of Pod %s was found in the cache", pod.Name)) + break + } + } - return usedPvc, nil + return usedPvc } diff --git a/images/sds-local-volume-scheduler-extender/pkg/scheduler/prioritize.go b/images/sds-local-volume-scheduler-extender/pkg/scheduler/prioritize.go index 77a59b35..0ad21dc5 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/scheduler/prioritize.go +++ b/images/sds-local-volume-scheduler-extender/pkg/scheduler/prioritize.go @@ -17,19 +17,18 @@ limitations under the License. package scheduler import ( - "context" "encoding/json" "errors" "fmt" "math" "net/http" + "sds-local-volume-scheduler-extender/pkg/cache" "sds-local-volume-scheduler-extender/pkg/logger" "sync" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" - "sigs.k8s.io/controller-runtime/pkg/client" ) func (s *scheduler) prioritize(w http.ResponseWriter, r *http.Request) { @@ -43,7 +42,7 @@ func (s *scheduler) prioritize(w http.ResponseWriter, r *http.Request) { return } - pvcs, err := getUsedPVC(s.ctx, s.client, input.Pod) + pvcs := getUsedPVC(s.log, s.cache, input.Pod) if err != nil { s.log.Error(err, "[prioritize] unable to get PVC from the Pod") http.Error(w, "bad request", http.StatusBadRequest) @@ -72,7 +71,7 @@ func (s *scheduler) prioritize(w http.ResponseWriter, r *http.Request) { s.log.Debug("[filter] successfully extracted the pvcRequests size") s.log.Debug("[prioritize] starts to score the nodes") - result, err := scoreNodes(s.ctx, s.client, s.log, input.Nodes, pvcs, scs, pvcRequests, s.defaultDivisor) + result, err := scoreNodes(s.log, s.cache, input.Nodes, pvcs, scs, pvcRequests, s.defaultDivisor) if err != nil { s.log.Error(err, "[prioritize] unable to score nodes") http.Error(w, "Bad Request.", http.StatusBadRequest) @@ -90,20 +89,15 @@ func (s *scheduler) prioritize(w http.ResponseWriter, r *http.Request) { } func scoreNodes( - ctx context.Context, - cl client.Client, log logger.Logger, + schedulerCache *cache.Cache, nodes *corev1.NodeList, - pvcs map[string]corev1.PersistentVolumeClaim, - scs map[string]v1.StorageClass, + pvcs map[string]*corev1.PersistentVolumeClaim, + scs map[string]*v1.StorageClass, pvcRequests map[string]PVCRequest, divisor float64, ) ([]HostPriority, error) { - lvgs, err := GetLVMVolumeGroups(ctx, cl) - if err != nil { - return nil, err - } - + lvgs := schedulerCache.GetAllLVG() scLVGs, err := GetSortedLVGsFromStorageClasses(scs) if err != nil { return nil, err @@ -152,13 +146,17 @@ func scoreNodes( lvg := lvgs[commonLVG.Name] switch pvcReq.DeviceType { case thick: - freeSpace, err = getVGFreeSpace(&lvg) + freeSpace, err = getVGFreeSpace(lvg) if err != nil { errs <- err return } - log.Trace(fmt.Sprintf("[scoreNodes] LVMVolumeGroup %s free thick space %s", lvg.Name, freeSpace.String())) - + log.Trace(fmt.Sprintf("[scoreNodes] LVMVolumeGroup %s free thick space before PVC reservation: %s", lvg.Name, freeSpace.String())) + reserved := schedulerCache.GetLVGReservedSpace(lvg.Name) + log.Trace(fmt.Sprintf("[scoreNodes] LVG %s PVC Space reservation: %s", lvg.Name, resource.NewQuantity(reserved, resource.BinarySI))) + spaceWithReserved := freeSpace.Value() - reserved + freeSpace = *resource.NewQuantity(spaceWithReserved, resource.BinarySI) + log.Trace(fmt.Sprintf("[scoreNodes] LVMVolumeGroup %s free thick space after PVC reservation: %s", lvg.Name, freeSpace.String())) case thin: thinPool := findMatchedThinPool(lvg.Status.ThinPools, commonLVG.Thin.PoolName) if thinPool == nil { @@ -173,8 +171,8 @@ func scoreNodes( errs <- err return } - } + lvgTotalSize, err := resource.ParseQuantity(lvg.Status.VGSize) if err != nil { errs <- err diff --git a/images/sds-local-volume-scheduler-extender/pkg/scheduler/route.go b/images/sds-local-volume-scheduler-extender/pkg/scheduler/route.go index b5d1951a..f7c52128 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/scheduler/route.go +++ b/images/sds-local-volume-scheduler-extender/pkg/scheduler/route.go @@ -83,25 +83,27 @@ func (s *scheduler) getCache(w http.ResponseWriter, r *http.Request) { nodeName string }) - lvgs := s.cache.GetLVGNames() - s.log.Info(fmt.Sprintf("LVG from cache: %v", lvgs)) + lvgs := s.cache.GetAllLVG() + //s.log.Info(fmt.Sprintf("LVG from cache: %v", lvgs)) for _, lvg := range lvgs { - pvcs := s.cache.GetAllPVCByLVG(lvg) - s.log.Info(fmt.Sprintf("LVG %s has PVC from cache: %v", lvg, pvcs)) + pvcs := s.cache.GetAllPVCByLVG(lvg.Name) + //for _, pvc := range pvcs { + // s.log.Trace(fmt.Sprintf("LVG %s has PVC from cache: %v", lvg, pvc.Name)) + //} - result[lvg] = make([]struct { + result[lvg.Name] = make([]struct { pvcName string nodeName string }, 0) for _, pvc := range pvcs { - result[lvg] = append(result[lvg], struct { + result[lvg.Name] = append(result[lvg.Name], struct { pvcName string nodeName string - }{pvcName: pvc.Name, nodeName: s.cache.GetPVCNodeName(pvc.Name)}) + }{pvcName: pvc.Name, nodeName: s.cache.GetPVCNodeName(lvg.Name, pvc)}) } } - s.log.Info(fmt.Sprintf("Result len: %d", len(result))) + //s.log.Info(fmt.Sprintf("Result len: %d", len(result))) _, err := w.Write([]byte(fmt.Sprintf("%+v", result))) if err != nil { From 0ed85a14d6600546e76ceac7c338736f31c2a3ab Mon Sep 17 00:00:00 2001 From: Viktor Kramarenko Date: Tue, 2 Apr 2024 12:24:24 +0300 Subject: [PATCH 11/21] added more logs Signed-off-by: Viktor Kramarenko --- .../cmd/cmd/root.go | 5 -- .../pkg/cache/cache.go | 12 +-- .../pkg/controller/lvg_watcher_cache.go | 34 +++++--- .../pkg/controller/pvc_watcher_cache.go | 87 +++++++++++-------- .../pkg/scheduler/filter.go | 39 ++++++--- 5 files changed, 101 insertions(+), 76 deletions(-) diff --git a/images/sds-local-volume-scheduler-extender/cmd/cmd/root.go b/images/sds-local-volume-scheduler-extender/cmd/cmd/root.go index 384e596e..219fa6ec 100644 --- a/images/sds-local-volume-scheduler-extender/cmd/cmd/root.go +++ b/images/sds-local-volume-scheduler-extender/cmd/cmd/root.go @@ -137,11 +137,6 @@ func subMain(parentCtx context.Context) error { return err } - //cl, err := client.New(kConfig, client.Options{ - // Scheme: scheme, - // WarningHandler: client.WarningHandlerOptions{}, - //}) - schedulerCache := cache.NewCache(config.CacheSize) log.Info("[subMain] scheduler cache was initialized") log.Debug(fmt.Sprintf("[subMain] scheduler cache struct: %+v", schedulerCache)) diff --git a/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go b/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go index ee309b4f..f31e4570 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go +++ b/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go @@ -100,7 +100,7 @@ func (c *Cache) DeleteLVG(lvgName string) { delete(c.lvgs, lvgName) } -func (c *Cache) AddPVCToFilterQueue(pvc *v1.PersistentVolumeClaim) { +func (c *Cache) AddPVCToCacheQueue(pvc *v1.PersistentVolumeClaim) { pvcKey := configurePVCKey(pvc) c.pvcQueue[pvcKey] = pvc } @@ -133,16 +133,6 @@ func (c *Cache) UpdatePVC(lvgName string, pvc *v1.PersistentVolumeClaim) error { return nil } -func (c *Cache) TryGetPVC(lvgName string, pvc *v1.PersistentVolumeClaim) *v1.PersistentVolumeClaim { - pvcKey := configurePVCKey(pvc) - - if c.lvgs[lvgName] == nil { - return nil - } - - return c.lvgs[lvgName].pvcs[pvcKey].pvc -} - func (c *Cache) CheckPVCInQueue(pvc *v1.PersistentVolumeClaim) bool { pvcKey := configurePVCKey(pvc) diff --git a/images/sds-local-volume-scheduler-extender/pkg/controller/lvg_watcher_cache.go b/images/sds-local-volume-scheduler-extender/pkg/controller/lvg_watcher_cache.go index 7329511f..ca10a229 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/controller/lvg_watcher_cache.go +++ b/images/sds-local-volume-scheduler-extender/pkg/controller/lvg_watcher_cache.go @@ -27,8 +27,7 @@ func RunLVGWatcherCacheController( log logger.Logger, cache *cache.Cache, ) (controller.Controller, error) { - log.Info("[RunLVGWatcherCacheController] starts the work WITH EVENTS") - //cl := mgr.GetClient() + log.Info("[RunLVGWatcherCacheController] starts the work") c, err := controller.New(LVGWatcherCacheCtrlName, mgr, controller.Options{ Reconciler: reconcile.Func(func(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { @@ -56,12 +55,10 @@ func RunLVGWatcherCacheController( return } - // PopulateLVG log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] tries to get the LVMVolumeGroup %s from the cache", lvg.Name)) existedLVG := cache.TryGetLVG(lvg.Name) - if existedLVG != nil { - log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] the LVMVolumeGroup %s was found. It will be updated", lvg.Name)) + log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] the LVMVolumeGroup %s was found in the cache. It will be updated", lvg.Name)) err = cache.UpdateLVG(lvg) if err != nil { log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to update the LVMVolumeGroup %s cache", lvg.Name)) @@ -69,7 +66,7 @@ func RunLVGWatcherCacheController( log.Info(fmt.Sprintf("[RunLVGWatcherCacheController] cache was updated for the LVMVolumeGroup %s", lvg.Name)) } else { - log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] the LVMVolumeGroup %s was not found. It will be added", lvg.Name)) + log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] the LVMVolumeGroup %s was not found. It will be added to the cache", lvg.Name)) err = cache.AddLVG(lvg) if err != nil { log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to add the LVMVolumeGroup %s to the cache", lvg.Name)) @@ -78,16 +75,19 @@ func RunLVGWatcherCacheController( log.Info(fmt.Sprintf("[RunLVGWatcherCacheController] cache was added for the LVMVolumeGroup %s", lvg.Name)) } + log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] starts to clear the cache for the LVMVolumeGroup %s", lvg.Name)) pvcs := cache.GetAllPVCByLVG(lvg.Name) for _, pvc := range pvcs { + log.Trace(fmt.Sprintf("[RunLVGWatcherCacheController] cached PVC %s belongs to LVMVolumeGroup %s", pvc.Name, lvg.Name)) if pvc.Status.Phase == v1.ClaimBound { + log.Trace(fmt.Sprintf("[RunLVGWatcherCacheController] cached PVC %s has Status.Phase Bound. It will be removed from the cache for LVMVolumeGroup %s", pvc.Name, lvg.Name)) err = cache.RemoveBoundedPVCSpaceReservation(lvg.Name, pvc) if err != nil { - log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to remove PVC %s from the cache for the LVG %s", pvc.Name, lvg.Name)) + log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to remove PVC %s from the cache for the LVMVolumeGroup %s", pvc.Name, lvg.Name)) continue } - log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] PVC %s was removed from the LVG %s in the cache", pvc.Name, lvg.Name)) + log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] PVC %s was removed from the cache for LVMVolumeGroup %s", pvc.Name, lvg.Name)) } } @@ -109,47 +109,53 @@ func RunLVGWatcherCacheController( return } + log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] starts to calculate the size difference for LVMVolumeGroup %s", newLvg.Name)) oldSize, err := resource.ParseQuantity(oldLvg.Status.AllocatedSize) if err != nil { log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to parse the allocated size for the LVMVolumeGroup %s", oldLvg.Name)) return } + log.Trace(fmt.Sprintf("[RunLVGWatcherCacheController] old state LVMVolumeGroup %s has size %s", oldLvg.Name, oldSize.String())) newSize, err := resource.ParseQuantity(newLvg.Status.AllocatedSize) if err != nil { log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to parse the allocated size for the LVMVolumeGroup %s", oldLvg.Name)) return } + log.Trace(fmt.Sprintf("[RunLVGWatcherCacheController] new state LVMVolumeGroup %s has size %s", newLvg.Name, newSize.String())) + log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] successfully calculated the size difference for LVMVolumeGroup %s", newLvg.Name)) if newLvg.DeletionTimestamp != nil || oldSize.Value() == newSize.Value() { - log.Info(fmt.Sprintf("[RunLVGWatcherCacheController] the LVMVolumeGroup %s should not be reconciled", newLvg.Name)) + log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] the LVMVolumeGroup %s should not be reconciled", newLvg.Name)) return } + log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] the LVMVolumeGroup %s should be reconciled by Update Func. It will be updated in the cache", newLvg.Name)) err = cache.UpdateLVG(newLvg) if err != nil { log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to update the LVMVolumeGroup %s cache", newLvg.Name)) return } + log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] successfully updated the LVMVolumeGroup %s in the cache", newLvg.Name)) cachedPvcs := cache.GetAllPVCByLVG(newLvg.Name) for _, pvc := range cachedPvcs { - log.Trace(fmt.Sprintf("[RunLVGWatcherCacheController] PVC %s from the cache belongs to LVG %s", pvc.Name, newLvg.Name)) + log.Trace(fmt.Sprintf("[RunLVGWatcherCacheController] PVC %s from the cache belongs to LVMVolumeGroup %s", pvc.Name, newLvg.Name)) log.Trace(fmt.Sprintf("[RunLVGWatcherCacheController] PVC %s has status phase %s", pvc.Name, pvc.Status.Phase)) if pvc.Status.Phase == v1.ClaimBound { - log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] PVC %s from the cache has status phase Bound. It will be removed from the reserved space of LVG %s", pvc.Name, newLvg.Name)) + log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] PVC %s from the cache has Status.Phase Bound. It will be removed from the reserved space in the LVMVolumeGroup %s", pvc.Name, newLvg.Name)) err = cache.RemoveBoundedPVCSpaceReservation(newLvg.Name, pvc) if err != nil { - log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to remove PVC %s from the cache for the LVG %s", pvc.Name, newLvg.Name)) + log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to remove PVC %s from the cache in the LVMVolumeGroup %s", pvc.Name, newLvg.Name)) continue } - log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] PVC %s was removed from the LVG %s in the cache", pvc.Name, newLvg.Name)) + log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] PVC %s was removed from the LVMVolumeGroup %s in the cache", pvc.Name, newLvg.Name)) } } - log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] reconciled LVMVolumeGroup %s cache", newLvg.Name)) + log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] Update Func ends reconciliation the LVMVolumeGroup %s cache", newLvg.Name)) }, DeleteFunc: func(ctx context.Context, e event.DeleteEvent, q workqueue.RateLimitingInterface) { log.Info(fmt.Sprintf("[RunCacheWatcherController] DeleteFunc starts the cache reconciliation for the LVMVolumeGroup %s", e.Object.GetName())) diff --git a/images/sds-local-volume-scheduler-extender/pkg/controller/pvc_watcher_cache.go b/images/sds-local-volume-scheduler-extender/pkg/controller/pvc_watcher_cache.go index 9781c8f5..e06df377 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/controller/pvc_watcher_cache.go +++ b/images/sds-local-volume-scheduler-extender/pkg/controller/pvc_watcher_cache.go @@ -22,7 +22,7 @@ func RunPVCWatcherCacheController( log logger.Logger, schedulerCache *cache.Cache, ) error { - log.Info("[RunPVCWatcherCacheController] starts the work WITH EVENTS") + log.Info("[RunPVCWatcherCacheController] starts the work") inf, err := mgr.GetCache().GetInformer(ctx, &v1.PersistentVolumeClaim{}) if err != nil { @@ -43,45 +43,53 @@ func RunPVCWatcherCacheController( switch shouldAddPVCToCache(schedulerCache, pvc) { case true: // Добавляем в queue, иначе фильтр не сможет получить ее из кеша - schedulerCache.AddPVCToFilterQueue(pvc) - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s was added to the cache", pvc.Name)) + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s should be added to the cache", pvc.Name)) + schedulerCache.AddPVCToCacheQueue(pvc) + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s was added to the cache queue", pvc.Name)) case false: - // Update Func (если рухнули на апдейте) - selectedNode, wasSelected := pvc.Annotations[cache.SelectedNodeAnnotation] + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s was found in the cache queue", pvc.Name)) + selectedNodeName, wasSelected := pvc.Annotations[cache.SelectedNodeAnnotation] if !wasSelected { log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s should not be reconciled by Add Func due it has not selected node annotation", pvc.Name)) return } - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s has node annotation, it will be reconciled in Add func", pvc.Name)) + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s has selected node annotation, it will be reconciled in Add func", pvc.Name)) - //cachedPvc := schedulerCache.TryGetPVC(pvc) - //if cachedPvc == nil { - // log.Error(fmt.Errorf("PVC %s was not found in the cache", pvc.Name), fmt.Sprintf("[RunPVCWatcherCacheController] unable to get PVC %s from the cache", pvc.Name)) - // return - //} - - lvgsOnTheNode := schedulerCache.GetLVGNamesByNodeName(selectedNode) + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] starts to find common LVMVolumeGroup for the selected node %s and PVC %s", selectedNodeName, pvc.Name)) + log.Trace(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s has been selected to the node %s", pvc.Name, selectedNodeName)) + lvgsOnTheNode := schedulerCache.GetLVGNamesByNodeName(selectedNodeName) + for _, lvgName := range lvgsOnTheNode { + log.Trace(fmt.Sprintf("[RunPVCWatcherCacheController] LVMVolumeGroup %s belongs to the node %s", lvgName, selectedNodeName)) + } lvgsForPVC := schedulerCache.GetLVGNamesForPVC(pvc) + for _, lvgName := range lvgsForPVC { + log.Trace(fmt.Sprintf("[RunPVCWatcherCacheController] LVMVolumeGroup %s belongs to PVC %s", lvgName, pvc.Name)) + } - var lvgName string + var commonLVGName string for _, pvcLvg := range lvgsForPVC { if slices.Contains(lvgsOnTheNode, pvcLvg) { - lvgName = pvcLvg + commonLVGName = pvcLvg } } + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] successfully found common LVMVolumeGroup %s for the selected node %s and PVC %s", commonLVGName, selectedNodeName, pvc.Name)) - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s has status phase: %s", pvc.Name, pvc.Status.Phase)) - err = schedulerCache.UpdatePVC(lvgName, pvc) + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] starts to update PVC %s in the cache", pvc.Name)) + log.Trace(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s has status phase: %s", pvc.Name, pvc.Status.Phase)) + err = schedulerCache.UpdatePVC(commonLVGName, pvc) if err != nil { log.Error(err, fmt.Sprintf("[RunPVCWatcherCacheController] unable to update PVC %s in the cache", pvc.Name)) return } + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] successfully updated PVC %s in the cache", pvc.Name)) // У PVC выбралась нода, но она еще не в баунд (в кеше PVC без ноды на лвгхах) + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] starts to remove unbound PVC %s space reservation from the cache", pvc.Name)) err = schedulerCache.RemoveUnboundedPVCSpaceReservation(log, pvc) if err != nil { log.Error(err, fmt.Sprintf("[RunPVCWatcherCacheController] unable to remove space reservation in the cache for unbounded PVC %s", pvc.Name)) } + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] succefully ended the unbound PVC %s space reservation from the cache", pvc.Name)) } }, UpdateFunc: func(oldObj, newObj interface{}) { @@ -93,37 +101,48 @@ func RunPVCWatcherCacheController( } log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] UpdateFunc starts the reconciliation for the PVC %s", pvc.Name)) - selectedNode, wasSelected := pvc.Annotations[cache.SelectedNodeAnnotation] + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s was found in the cache queue", pvc.Name)) + selectedNodeName, wasSelected := pvc.Annotations[cache.SelectedNodeAnnotation] if !wasSelected { log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s should not be reconciled by Add Func due it has not selected node annotation", pvc.Name)) return } - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s has node annotation, it will be reconciled in Update func", pvc.Name)) + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s has selected node annotation, it will be reconciled in Add func", pvc.Name)) - lvgsOnTheNode := schedulerCache.GetLVGNamesByNodeName(selectedNode) + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] starts to find common LVMVolumeGroup for the selected node %s and PVC %s", selectedNodeName, pvc.Name)) + log.Trace(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s has been selected to the node %s", pvc.Name, selectedNodeName)) + lvgsOnTheNode := schedulerCache.GetLVGNamesByNodeName(selectedNodeName) + for _, lvgName := range lvgsOnTheNode { + log.Trace(fmt.Sprintf("[RunPVCWatcherCacheController] LVMVolumeGroup %s belongs to the node %s", lvgName, selectedNodeName)) + } lvgsForPVC := schedulerCache.GetLVGNamesForPVC(pvc) + for _, lvgName := range lvgsForPVC { + log.Trace(fmt.Sprintf("[RunPVCWatcherCacheController] LVMVolumeGroup %s belongs to PVC %s", lvgName, pvc.Name)) + } - var lvgName string + var commonLVGName string for _, pvcLvg := range lvgsForPVC { if slices.Contains(lvgsOnTheNode, pvcLvg) { - lvgName = pvcLvg + commonLVGName = pvcLvg } } - log.Trace(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s uses LVG %s on node %s", pvc.Name, lvgName, selectedNode)) + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] successfully found common LVMVolumeGroup %s for the selected node %s and PVC %s", commonLVGName, selectedNodeName, pvc.Name)) + + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] starts to update PVC %s in the cache", pvc.Name)) log.Trace(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s has status phase: %s", pvc.Name, pvc.Status.Phase)) - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] updates cache PVC %s in LVG %s", pvc.Name, lvgName)) - err = schedulerCache.UpdatePVC(lvgName, pvc) + err = schedulerCache.UpdatePVC(commonLVGName, pvc) if err != nil { log.Error(err, fmt.Sprintf("[RunPVCWatcherCacheController] unable to update PVC %s in the cache", pvc.Name)) return } - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] successfully updated cache PVC %s in LVG %s", pvc.Name, lvgName)) + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] successfully updated PVC %s in the cache", pvc.Name)) - // У PVC выбралась нода, но она еще не в баунд (в кеше PVC без ноды на лвгхах) + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] starts to remove unbound PVC %s space reservation from the cache", pvc.Name)) err = schedulerCache.RemoveUnboundedPVCSpaceReservation(log, pvc) if err != nil { log.Error(err, fmt.Sprintf("[RunPVCWatcherCacheController] unable to remove space reservation in the cache for unbounded PVC %s", pvc.Name)) } + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] succefully ended the unbound PVC %s space reservation from the cache", pvc.Name)) }, DeleteFunc: func(obj interface{}) { log.Info("[RunPVCWatcherCacheController] Delete Func reconciliation starts") @@ -134,8 +153,9 @@ func RunPVCWatcherCacheController( } log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] DeleteFunc starts the reconciliation for the PVC %s", pvc.Name)) + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s was removed from the cluster. It will be force deleted from the cache", pvc.Name)) schedulerCache.RemovePVCSpaceReservationForced(pvc) - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s was force removed from the cache", pvc.Name)) + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] successfully force removed PVC %s from the cache", pvc.Name)) }, }) if err != nil { @@ -146,17 +166,16 @@ func RunPVCWatcherCacheController( } func shouldAddPVCToCache(schedulerCache *cache.Cache, pvc *v1.PersistentVolumeClaim) bool { - //_, selected := pvc.Annotations[cache.SelectedNodeAnnotation] - if pvc.Status.Phase != v1.ClaimBound { - return true + if pvc.Status.Phase == v1.ClaimBound { + return false } exist := schedulerCache.CheckPVCInQueue(pvc) - if !exist { - return true + if exist { + return false } - return false + return true } // diff --git a/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go b/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go index 975b4b45..1dc6a387 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go +++ b/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go @@ -226,6 +226,9 @@ func filterNodes( } lvgs := schedulerCache.GetAllLVG() + for _, lvg := range lvgs { + log.Trace(fmt.Sprintf("[filterNodes] LVMVolumeGroup %s in the cache", lvg.Name)) + } lvgsThickFree, err := getLVGThickFreeSpaces(lvgs) if err != nil { @@ -236,19 +239,29 @@ func filterNodes( for lvgName, freeSpace := range lvgsThickFree { log.Trace(fmt.Sprintf("[filterNodes] current LVG %s Thick free space %s", lvgName, resource.NewQuantity(freeSpace, resource.BinarySI))) reservedSize := schedulerCache.GetLVGReservedSpace(lvgName) - log.Trace(fmt.Sprintf("[filterNodes] current LVG %s reserved by pvc from cache space %s", lvgName, resource.NewQuantity(reservedSize, resource.BinarySI))) + log.Trace(fmt.Sprintf("[filterNodes] current LVG %s reserved PVC space %s", lvgName, resource.NewQuantity(reservedSize, resource.BinarySI))) lvgsThickFree[lvgName] -= reservedSize } log.Trace(fmt.Sprintf("[filterNodes] current LVGs Thick FreeSpace with reserved PVC: %+v", lvgsThickFree)) lvgsThickFreeMutex := &sync.RWMutex{} + log.Debug("[filterNodes] starts to get LVMVolumeGroups for Storage Classes") scLVGs, err := GetSortedLVGsFromStorageClasses(scs) if err != nil { return nil, err } + log.Debug("[filterNodes] successfully got LVMVolumeGroups for Storage Classes") + for scName, sortedLVGs := range scLVGs { + for _, lvg := range sortedLVGs { + log.Trace(fmt.Sprintf("[filterNodes] LVMVolumeGroup %s belongs to Storage Class %s", lvg.Name, scName)) + } + } usedLVGs := RemoveUnusedLVGs(lvgs, scLVGs) + for _, lvg := range usedLVGs { + log.Trace(fmt.Sprintf("[filterNodes] the LVMVolumeGroup %s is actually used", lvg.Name)) + } nodeLVGs := SortLVGsByNodeName(usedLVGs) for n, ls := range nodeLVGs { @@ -259,14 +272,14 @@ func filterNodes( commonNodes, err := getCommonNodesByStorageClasses(scs, nodeLVGs) for nodeName := range commonNodes { - log.Trace(fmt.Sprintf("[filterNodes] common node %s", nodeName)) + log.Trace(fmt.Sprintf("[filterNodes] Node %s is a common for every storage class", nodeName)) } result := &ExtenderFilterResult{ Nodes: &corev1.NodeList{}, FailedNodes: FailedNodesMap{}, } - failedNodesMapMutex := &sync.Mutex{} + failedNodesMutex := &sync.Mutex{} wg := &sync.WaitGroup{} wg.Add(len(nodes.Items)) @@ -282,9 +295,9 @@ func filterNodes( if _, common := commonNodes[node.Name]; !common { log.Debug(fmt.Sprintf("[filterNodes] node %s is not common for used Storage Classes", node.Name)) - failedNodesMapMutex.Lock() + failedNodesMutex.Lock() result.FailedNodes[node.Name] = "node is not common for used Storage Classes" - failedNodesMapMutex.Unlock() + failedNodesMutex.Unlock() return } @@ -346,9 +359,9 @@ func filterNodes( } if !hasEnoughSpace { - failedNodesMapMutex.Lock() + failedNodesMutex.Lock() result.FailedNodes[node.Name] = "not enough space" - failedNodesMapMutex.Unlock() + failedNodesMutex.Unlock() return } @@ -611,8 +624,13 @@ func getUsedPVC(log logger.Logger, schedulerCache *cache.Cache, pod *corev1.Pod) for { pvcMap := schedulerCache.GetPVCsFromQueue(pod.Namespace) + for _, pvc := range pvcMap { + log.Trace(fmt.Sprintf("[getUsedPVC] PVC %s from namespace %s was found in the cache", pvc.Name, pod.Namespace)) + } + for _, volume := range pod.Spec.Volumes { if volume.PersistentVolumeClaim != nil { + log.Trace(fmt.Sprintf("[getUsedPVC] Pod %s uses PVC %s", pod.Name, volume.PersistentVolumeClaim.ClaimName)) usedPvc[volume.PersistentVolumeClaim.ClaimName] = pvcMap[volume.PersistentVolumeClaim.ClaimName] } } @@ -620,15 +638,12 @@ func getUsedPVC(log logger.Logger, schedulerCache *cache.Cache, pod *corev1.Pod) filled := false if len(pvcMap) > 0 { filled = true - for _, pvc := range pvcMap { - log.Trace(fmt.Sprintf("[getUsedPVC] PVC %s from the cache of namespace %s", pvc.Name, pod.Namespace)) - } for _, volume := range pod.Spec.Volumes { if volume.PersistentVolumeClaim != nil { if _, added := usedPvc[volume.PersistentVolumeClaim.ClaimName]; !added { filled = false - log.Warning(fmt.Sprintf("[getUsedPVC] ClaimName %s was not found in the cache for Pod %s", volume.PersistentVolumeClaim.ClaimName, pod.Name)) + log.Warning(fmt.Sprintf("[getUsedPVC] PVC %s was not found in the cache for Pod %s", volume.PersistentVolumeClaim.ClaimName, pod.Name)) break } } @@ -642,7 +657,7 @@ func getUsedPVC(log logger.Logger, schedulerCache *cache.Cache, pod *corev1.Pod) } if filled { - log.Debug(fmt.Sprintf("[getUsedPVC] Every claim of Pod %s was found in the cache", pod.Name)) + log.Debug(fmt.Sprintf("[getUsedPVC] Every PVC for Pod %s was found in the cache", pod.Name)) break } From 225888b68f231eb24ffb8201324cbb59c50161b5 Mon Sep 17 00:00:00 2001 From: Aleksandr Zimin Date: Wed, 3 Apr 2024 00:08:44 +0300 Subject: [PATCH 12/21] bump go deps Signed-off-by: Aleksandr Zimin --- images/sds-local-volume-csi/go.mod | 58 +++++++++++++++--------------- images/sds-local-volume-csi/go.sum | 54 ++++++++++++++++++++++++++-- 2 files changed, 82 insertions(+), 30 deletions(-) diff --git a/images/sds-local-volume-csi/go.mod b/images/sds-local-volume-csi/go.mod index 1df6e911..60c48d2c 100644 --- a/images/sds-local-volume-csi/go.mod +++ b/images/sds-local-volume-csi/go.mod @@ -5,54 +5,56 @@ go 1.21 require ( github.com/container-storage-interface/spec v1.9.0 github.com/go-logr/logr v1.4.1 - github.com/golang/protobuf v1.5.3 + github.com/golang/protobuf v1.5.4 golang.org/x/sync v0.6.0 - google.golang.org/grpc v1.60.1 - gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.29.0 - k8s.io/apiextensions-apiserver v0.29.0 - k8s.io/apimachinery v0.29.0 - k8s.io/client-go v0.29.0 - k8s.io/klog/v2 v2.110.1 - k8s.io/mount-utils v0.29.1 - k8s.io/utils v0.0.0-20230726121419-3b25d923346b - sigs.k8s.io/controller-runtime v0.17.0 + google.golang.org/grpc v1.62.1 + gopkg.in/yaml.v2 v2.4.0 + k8s.io/api v0.29.3 + k8s.io/apiextensions-apiserver v0.29.3 + k8s.io/apimachinery v0.29.3 + k8s.io/client-go v0.29.3 + k8s.io/klog/v2 v2.120.1 + k8s.io/mount-utils v0.29.3 + k8s.io/utils v0.0.0-20240310230437-4693a0247e57 + sigs.k8s.io/controller-runtime v0.17.2 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/evanphx/json-patch/v5 v5.8.0 // indirect - github.com/go-openapi/jsonpointer v0.19.6 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.22.3 // indirect + github.com/emicklei/go-restful/v3 v3.12.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/uuid v1.3.1 // indirect - github.com/imdario/mergo v0.3.6 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/imdario/mergo v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/moby/sys/mountinfo v0.6.2 // indirect + github.com/moby/sys/mountinfo v0.7.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/spf13/pflag v1.0.5 // indirect - golang.org/x/net v0.20.0 // indirect - golang.org/x/oauth2 v0.13.0 // indirect - golang.org/x/sys v0.16.0 // indirect - golang.org/x/term v0.16.0 // indirect + golang.org/x/net v0.22.0 // indirect + golang.org/x/oauth2 v0.18.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/time v0.3.0 // indirect + golang.org/x/time v0.5.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240108191215-35c7eff3a6b1 // indirect - google.golang.org/protobuf v1.32.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/kube-openapi v0.0.0-20240322212309-b815d8309940 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) + +replace github.com/imdario/mergo => github.com/imdario/mergo v0.3.16 diff --git a/images/sds-local-volume-csi/go.sum b/images/sds-local-volume-csi/go.sum index 3e846a59..aee18791 100644 --- a/images/sds-local-volume-csi/go.sum +++ b/images/sds-local-volume-csi/go.sum @@ -6,8 +6,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.12.0 h1:y2DdzBAURM29NFF94q6RaY4vjIH1rtwDapwQtU84iWk= +github.com/emicklei/go-restful/v3 v3.12.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= +github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -15,10 +19,16 @@ github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -27,6 +37,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -40,8 +52,10 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJY github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= +github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -59,6 +73,8 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= +github.com/moby/sys/mountinfo v0.7.1 h1:/tTvQaSJRr2FshkhXiIpux6fQ2Zvc4j7tAhMTStAG2g= +github.com/moby/sys/mountinfo v0.7.1/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -111,8 +127,12 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -128,10 +148,14 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -140,6 +164,8 @@ golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -155,12 +181,18 @@ google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAs google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto/googleapis/rpc v0.0.0-20240108191215-35c7eff3a6b1 h1:gphdwh0npgs8elJ4T6J+DQJHPVF7RsuJHCfwztUb4J4= google.golang.org/genproto/googleapis/rpc v0.0.0-20240108191215-35c7eff3a6b1/go.mod h1:daQN87bsDqDoe316QbbvX60nMoJQa4r6Ds0ZuoAe5yA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= +google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= +google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -174,22 +206,40 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.29.0 h1:NiCdQMY1QOp1H8lfRyeEf8eOwV6+0xA6XEE44ohDX2A= k8s.io/api v0.29.0/go.mod h1:sdVmXoz2Bo/cb77Pxi71IPTSErEW32xa4aXwKH7gfBA= +k8s.io/api v0.29.3 h1:2ORfZ7+bGC3YJqGpV0KSDDEVf8hdGQ6A03/50vj8pmw= +k8s.io/api v0.29.3/go.mod h1:y2yg2NTyHUUkIoTC+phinTnEa3KFM6RZ3szxt014a80= k8s.io/apiextensions-apiserver v0.29.0 h1:0VuspFG7Hj+SxyF/Z/2T0uFbI5gb5LRgEyUVE3Q4lV0= k8s.io/apiextensions-apiserver v0.29.0/go.mod h1:TKmpy3bTS0mr9pylH0nOt/QzQRrW7/h7yLdRForMZwc= +k8s.io/apiextensions-apiserver v0.29.3 h1:9HF+EtZaVpFjStakF4yVufnXGPRppWFEQ87qnO91YeI= +k8s.io/apiextensions-apiserver v0.29.3/go.mod h1:po0XiY5scnpJfFizNGo6puNU6Fq6D70UJY2Cb2KwAVc= k8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o= k8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMeis= +k8s.io/apimachinery v0.29.3 h1:2tbx+5L7RNvqJjn7RIuIKu9XTsIZ9Z5wX2G22XAa5EU= +k8s.io/apimachinery v0.29.3/go.mod h1:hx/S4V2PNW4OMg3WizRrHutyB5la0iCUbZym+W0EQIU= k8s.io/client-go v0.29.0 h1:KmlDtFcrdUzOYrBhXHgKw5ycWzc3ryPX5mQe0SkG3y8= k8s.io/client-go v0.29.0/go.mod h1:yLkXH4HKMAywcrD82KMSmfYg2DlE8mepPR4JGSo5n38= +k8s.io/client-go v0.29.3 h1:R/zaZbEAxqComZ9FHeQwOh3Y1ZUs7FaHKZdQtIc2WZg= +k8s.io/client-go v0.29.3/go.mod h1:tkDisCvgPfiRpxGnOORfkljmS+UrW+WtXAy2fTvXJB0= k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/kube-openapi v0.0.0-20240322212309-b815d8309940 h1:qVoMaQV5t62UUvHe16Q3eb2c5HPzLHYzsi0Tu/xLndo= +k8s.io/kube-openapi v0.0.0-20240322212309-b815d8309940/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/mount-utils v0.29.1 h1:veXlIm52Y4tm3H0pG03cOdkw0KOJxYDa0fQqhJCoqvQ= k8s.io/mount-utils v0.29.1/go.mod h1:9IWJTMe8tG0MYMLEp60xK9GYVeCdA3g4LowmnVi+t9Y= +k8s.io/mount-utils v0.29.3 h1:iEcqPP7Vv8UClH8nnMfovtmy/04fIloRW9JuSXykoZ0= +k8s.io/mount-utils v0.29.3/go.mod h1:9IWJTMe8tG0MYMLEp60xK9GYVeCdA3g4LowmnVi+t9Y= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20240310230437-4693a0247e57 h1:gbqbevonBh57eILzModw6mrkbwM0gQBEuevE/AaBsHY= +k8s.io/utils v0.0.0-20240310230437-4693a0247e57/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/controller-runtime v0.17.0 h1:fjJQf8Ukya+VjogLO6/bNX9HE6Y2xpsO5+fyS26ur/s= sigs.k8s.io/controller-runtime v0.17.0/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= +sigs.k8s.io/controller-runtime v0.17.2 h1:FwHwD1CTUemg0pW2otk7/U5/i5m2ymzvOXdbeGOUvw0= +sigs.k8s.io/controller-runtime v0.17.2/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= From 650465adac50531fef7c22af8c2b890cb44a706a Mon Sep 17 00:00:00 2001 From: Viktor Kramarenko Date: Thu, 4 Apr 2024 12:39:40 +0300 Subject: [PATCH 13/21] working version Signed-off-by: Viktor Kramarenko --- .../Dockerfile | 8 +- .../cmd/cmd/root.go | 5 +- .../go.mod | 6 +- .../go.sum | 8 +- .../pkg/cache/cache.go | 448 ++++++++++++++---- .../pkg/controller/lvg_watcher_cache.go | 24 +- .../pkg/controller/pvc_watcher_cache.go | 266 ++++++----- .../pkg/scheduler/filter.go | 105 ++-- .../pkg/scheduler/prioritize.go | 16 +- .../pkg/scheduler/route.go | 13 +- images/webhooks/src/go.mod | 2 +- 11 files changed, 601 insertions(+), 300 deletions(-) diff --git a/images/sds-local-volume-scheduler-extender/Dockerfile b/images/sds-local-volume-scheduler-extender/Dockerfile index cf5afcce..12ae60a2 100644 --- a/images/sds-local-volume-scheduler-extender/Dockerfile +++ b/images/sds-local-volume-scheduler-extender/Dockerfile @@ -1,7 +1,7 @@ -ARG BASE_GOLANG_20_ALPINE=registry.deckhouse.io/base_images/golang:1.20.4-alpine3.18@sha256:5f403dd08db2f0b40d4416e29d3080eec41cd6cf53a05d5e4bcece3a5c7a8ce6 -ARG BASE_GOLANG_20_ALPINE_BUILDER=$BASE_GOLANG_20_ALPINE +ARG BASE_GOLANG_22_ALPINE=registry.deckhouse.io/base_images/golang:1.22.1-alpine@sha256:0de6cf7cceab6ecbf0718bdfb675b08b78113c3709c5e4b99456cdb2ae8c2495 +ARG BASE_GOLANG_22_ALPINE_BUILDER=$BASE_GOLANG_22_ALPINE -FROM $BASE_GOLANG_20_ALPINE_BUILDER as builder +FROM $BASE_GOLANG_22_ALPINE_BUILDER as builder WORKDIR /go/src @@ -15,7 +15,7 @@ COPY . . WORKDIR /go/src/cmd RUN GOOS=linux GOARCH=amd64 go build -o sds-local-volume-scheduler-extender -FROM --platform=linux/amd64 $BASE_GOLANG_20_ALPINE +FROM --platform=linux/amd64 $BASE_GOLANG_22_ALPINE COPY --from=builder /go/src/cmd/sds-local-volume-scheduler-extender /go/src/cmd/sds-local-volume-scheduler-extender RUN apk add curl diff --git a/images/sds-local-volume-scheduler-extender/cmd/cmd/root.go b/images/sds-local-volume-scheduler-extender/cmd/cmd/root.go index 219fa6ec..cb872491 100644 --- a/images/sds-local-volume-scheduler-extender/cmd/cmd/root.go +++ b/images/sds-local-volume-scheduler-extender/cmd/cmd/root.go @@ -137,9 +137,8 @@ func subMain(parentCtx context.Context) error { return err } - schedulerCache := cache.NewCache(config.CacheSize) + schedulerCache := cache.NewCache(*log) log.Info("[subMain] scheduler cache was initialized") - log.Debug(fmt.Sprintf("[subMain] scheduler cache struct: %+v", schedulerCache)) h, err := scheduler.NewHandler(ctx, mgr.GetClient(), *log, schedulerCache, config.DefaultDivisor) if err != nil { @@ -153,7 +152,7 @@ func subMain(parentCtx context.Context) error { } log.Info(fmt.Sprintf("[subMain] successfully ran %s controller", controller.LVGWatcherCacheCtrlName)) - err = controller.RunPVCWatcherCacheController(ctx, mgr, *log, schedulerCache) + err = controller.RunPVCWatcherCacheController(mgr, *log, schedulerCache) if err != nil { log.Error(err, fmt.Sprintf("[subMain] unable to run %s controller", controller.PVCWatcherCacheCtrlName)) } diff --git a/images/sds-local-volume-scheduler-extender/go.mod b/images/sds-local-volume-scheduler-extender/go.mod index ec7b3c81..fee4dc85 100644 --- a/images/sds-local-volume-scheduler-extender/go.mod +++ b/images/sds-local-volume-scheduler-extender/go.mod @@ -6,14 +6,14 @@ require ( github.com/go-logr/logr v1.4.1 github.com/go-logr/zapr v1.3.0 github.com/spf13/cobra v1.8.0 - github.com/stretchr/testify v1.8.4 go.uber.org/zap v1.26.0 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.29.1 k8s.io/apimachinery v0.29.1 k8s.io/client-go v0.29.0 k8s.io/klog/v2 v2.110.1 - sigs.k8s.io/controller-runtime v0.17.0 + k8s.io/utils v0.0.0-20240310230437-4693a0247e57 + sigs.k8s.io/controller-runtime v0.17.2 sigs.k8s.io/yaml v1.4.0 ) @@ -44,7 +44,6 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.18.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect @@ -66,7 +65,6 @@ require ( k8s.io/apiextensions-apiserver v0.29.0 // indirect k8s.io/component-base v0.29.0 // indirect k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect - k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) diff --git a/images/sds-local-volume-scheduler-extender/go.sum b/images/sds-local-volume-scheduler-extender/go.sum index 51b1d86c..a22441c0 100644 --- a/images/sds-local-volume-scheduler-extender/go.sum +++ b/images/sds-local-volume-scheduler-extender/go.sum @@ -194,10 +194,10 @@ k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/controller-runtime v0.17.0 h1:fjJQf8Ukya+VjogLO6/bNX9HE6Y2xpsO5+fyS26ur/s= -sigs.k8s.io/controller-runtime v0.17.0/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= +k8s.io/utils v0.0.0-20240310230437-4693a0247e57 h1:gbqbevonBh57eILzModw6mrkbwM0gQBEuevE/AaBsHY= +k8s.io/utils v0.0.0-20240310230437-4693a0247e57/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.17.2 h1:FwHwD1CTUemg0pW2otk7/U5/i5m2ymzvOXdbeGOUvw0= +sigs.k8s.io/controller-runtime v0.17.2/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= diff --git a/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go b/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go index f31e4570..7658cf48 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go +++ b/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go @@ -3,8 +3,10 @@ package cache import ( "fmt" v1 "k8s.io/api/core/v1" + slices2 "k8s.io/utils/strings/slices" "sds-local-volume-scheduler-extender/api/v1alpha1" "sds-local-volume-scheduler-extender/pkg/logger" + "sync" ) const ( @@ -13,10 +15,10 @@ const ( ) type Cache struct { - lvgs map[string]*lvgCache - pvcQueue map[string]*v1.PersistentVolumeClaim - pvcLVGs map[string][]string - nodeLVGs map[string][]string + lvgs sync.Map + pvcLVGs sync.Map + nodeLVGs sync.Map + log logger.Logger } type lvgCache struct { @@ -25,39 +27,39 @@ type lvgCache struct { } type pvcCache struct { - pvc *v1.PersistentVolumeClaim - nodeName string + pvc *v1.PersistentVolumeClaim + selectedNode string } -func NewCache(size int) *Cache { +func NewCache(logger logger.Logger) *Cache { return &Cache{ - lvgs: make(map[string]*lvgCache, size), - pvcQueue: make(map[string]*v1.PersistentVolumeClaim, size), - pvcLVGs: make(map[string][]string, size), - nodeLVGs: make(map[string][]string, size), + log: logger, } } -func (c *Cache) AddLVG(lvg *v1alpha1.LvmVolumeGroup) error { - if _, exist := c.lvgs[lvg.Name]; exist { - return fmt.Errorf("the LVMVolumeGroup %s was found in the cache", lvg.Name) - } - - c.lvgs[lvg.Name] = &lvgCache{ +func (c *Cache) AddLVG(lvg *v1alpha1.LvmVolumeGroup) { + _, loaded := c.lvgs.LoadOrStore(lvg.Name, &lvgCache{ lvg: lvg, pvcs: make(map[string]*pvcCache, pvcPerLVGCount), + }) + if loaded { + c.log.Debug(fmt.Sprintf("[AddLVG] the LVMVolumeGroup %s has been already added to the cache", lvg.Name)) + return } for _, node := range lvg.Status.Nodes { - c.nodeLVGs[node.Name] = append(c.nodeLVGs[node.Name], lvg.Name) + lvgsOnTheNode, _ := c.nodeLVGs.Load(node.Name) + if lvgsOnTheNode == nil { + lvgsOnTheNode = make([]string, 0, 5) + } + lvgsOnTheNode = append(lvgsOnTheNode.([]string), lvg.Name) + c.nodeLVGs.Store(node.Name, lvgsOnTheNode) } - - return nil } func (c *Cache) UpdateLVG(lvg *v1alpha1.LvmVolumeGroup) error { - if cache, exist := c.lvgs[lvg.Name]; exist { - cache.lvg = lvg + if cache, found := c.lvgs.Load(lvg.Name); found { + cache.(*lvgCache).lvg = lvg return nil } @@ -65,160 +67,418 @@ func (c *Cache) UpdateLVG(lvg *v1alpha1.LvmVolumeGroup) error { } func (c *Cache) TryGetLVG(name string) *v1alpha1.LvmVolumeGroup { - if c.lvgs[name] == nil { + lvgCh, found := c.lvgs.Load(name) + if !found { + c.log.Debug(fmt.Sprintf("[TryGetLVG] the LVMVolumeGroup %s was not found in the cache. Return nil", name)) return nil } - return c.lvgs[name].lvg + return lvgCh.(*lvgCache).lvg } func (c *Cache) GetLVGNamesByNodeName(nodeName string) []string { - return c.nodeLVGs[nodeName] + lvgs, found := c.nodeLVGs.Load(nodeName) + if !found { + c.log.Debug(fmt.Sprintf("[GetLVGNamesByNodeName] no LVMVolumeGroup was found in the cache for the node %s. Return empty slice", nodeName)) + return []string{} + } + + return lvgs.([]string) } func (c *Cache) GetAllLVG() map[string]*v1alpha1.LvmVolumeGroup { - lvgs := make(map[string]*v1alpha1.LvmVolumeGroup, len(c.lvgs)) - for _, lvgCh := range c.lvgs { - lvgs[lvgCh.lvg.Name] = lvgCh.lvg - } + lvgs := make(map[string]*v1alpha1.LvmVolumeGroup) + c.lvgs.Range(func(lvgName, lvgCh any) bool { + if lvgCh.(*lvgCache).lvg == nil { + c.log.Error(fmt.Errorf("LVMVolumeGroup %s is not initialized", lvgName), fmt.Sprintf("[GetAllLVG] an error occurs while iterating the LVMVolumeGroups")) + return true + } + + lvgs[lvgName.(string)] = lvgCh.(*lvgCache).lvg + return true + }) return lvgs } -func (c *Cache) GetLVGReservedSpace(lvgName string) int64 { - lvg := c.lvgs[lvgName] +func (c *Cache) GetLVGReservedSpace(lvgName string) (int64, error) { + lvg, found := c.lvgs.Load(lvgName) + if !found { + c.log.Debug(fmt.Sprintf("[GetLVGReservedSpace] the LVMVolumeGroup %s was not found in the cache. Returns 0", lvgName)) + return 0, nil + } + if lvg.(*lvgCache).pvcs == nil { + err := fmt.Errorf("LVMVolumeGroup %s has no cached PVC", lvgName) + c.log.Error(err, fmt.Sprintf("[GetLVGReservedSpace] an error occurs for the LVMVolumeGroup %s", lvgName)) + return 0, err + } var space int64 - for _, pvc := range lvg.pvcs { + for _, pvc := range lvg.(*lvgCache).pvcs { space += pvc.pvc.Spec.Resources.Requests.Storage().Value() } - return space + return space, nil } func (c *Cache) DeleteLVG(lvgName string) { - delete(c.lvgs, lvgName) + c.lvgs.Delete(lvgName) } -func (c *Cache) AddPVCToCacheQueue(pvc *v1.PersistentVolumeClaim) { - pvcKey := configurePVCKey(pvc) - c.pvcQueue[pvcKey] = pvc -} +func (c *Cache) AddPVCToLVG(lvgName string, pvc *v1.PersistentVolumeClaim) error { + if pvc.Status.Phase == v1.ClaimBound { + c.log.Debug(fmt.Sprintf("[AddPVCToLVG] PVC %s/%s has status phase BOUND. It will not be added to the cache", pvc.Namespace, pvc.Name)) + return nil + } -func (c *Cache) AddUnboundedPVCToLVG(lvgName string, pvc *v1.PersistentVolumeClaim) error { + // this case will be triggered if the controller recovers after fail and finds some pending pvcs with selected nodes + c.log.Trace(fmt.Sprintf("[AddPVCToLVG] PVC %s/%s annotations: %v", pvc.Namespace, pvc.Name, pvc.Annotations)) if pvc.Annotations[SelectedNodeAnnotation] != "" { - return fmt.Errorf("PVC %s is expected to not have a selected node, but got one: %s", pvc.Name, pvc.Annotations[SelectedNodeAnnotation]) + c.log.Debug(fmt.Sprintf("PVC %s/%s has selected node anotation, selected node: %s", pvc.Namespace, pvc.Name, pvc.Annotations[SelectedNodeAnnotation])) + pvcKey := configurePVCKey(pvc) + + // если уже выбрана нода, то нам нужно проверить, что переданная лвг принадлежит этой ноде и если так, то мы просто обновим ее в ней, если не принаджелит, то скипаем с варнингом + lvgsOnTheNode, found := c.nodeLVGs.Load(pvc.Annotations[SelectedNodeAnnotation]) + if !found { + err := fmt.Errorf("no LVMVolumeGroups found for the node %s", pvc.Annotations[SelectedNodeAnnotation]) + c.log.Error(err, fmt.Sprintf("[AddPVCToLVG] an error occured while trying to add PVC %s to the cache", pvcKey)) + return err + } + + if slices2.Contains(lvgsOnTheNode.([]string), lvgName) { + c.log.Debug(fmt.Sprintf("[AddPVCToLVG] LVMVolumeGroup %s belongs to PVC %s/%s selected node %s", lvgName, pvc.Namespace, pvc.Name, pvc.Annotations[SelectedNodeAnnotation])) + lvgCh, found := c.lvgs.Load(lvgName) + if !found { + err := fmt.Errorf("the LVMVolumeGroup %s was not found in the cache", lvgName) + c.log.Error(err, fmt.Sprintf("[AddPVCToLVG] an error occured while trying to add PVC %s to the cache", pvcKey)) + return err + } + + pvcCh, found := lvgCh.(*lvgCache).pvcs[pvcKey] + if found { + c.log.Debug(fmt.Sprintf("[AddPVCToLVG] PVC %s cache has been already added to the LVMVolumeGroup %s. It will be updated", pvcKey, lvgName)) + pvcCh.pvc = pvc + } else { + c.log.Debug(fmt.Sprintf("[AddPVCToLVG] PVC %s cache was not found in LVMVolumeGroup %s. It will be added", pvcKey, lvgName)) + c.addNewPVC(lvgCh.(*lvgCache), pvc, lvgName) + } + } else { + c.log.Debug(fmt.Sprintf("[AddPVCToLVG] LVMVolumeGroup %s does not belong to PVC %s/%s selected node %s. It will be skipped", lvgName, pvc.Namespace, pvc.Name, pvc.Annotations[SelectedNodeAnnotation])) + } + + //lvgsForPVC, found := c.pvcLVGs.Load(pvcKey) + //if !found { + // c.log.Warning(fmt.Sprintf("LVMVolumeGroups were not found in the cache for PVC %s. The selected node check will be skipped, and ", pvcKey)) + // return err + //} + // + //c.log.Trace(fmt.Sprintf("[AddPVCToLVG] PVC %s has %d LVMVolumeGroups in the cache: %s", pvcKey, len(lvgsForPVC.([]string)), strings.Join(lvgsForPVC.([]string), ","))) + //for _, pvcLvgName := range lvgsForPVC.([]string) { + // lvgCh, found := c.lvgs.Load(pvcLvgName) + // if !found { + // err := fmt.Errorf("LVMVolumeGroup %s was not found in the cache", pvcLvgName) + // c.log.Error(err, fmt.Sprintf("[AddPVCToLVG] an error occured while trying to add PVC %s to the cache", pvcKey)) + // return err + // } + // + // if pvcCh, exist := lvgCh.(*lvgCache).pvcs[pvcKey]; exist { + // if pvcCh == nil { + // err := fmt.Errorf("cache struct is not initialized for PVC %s", pvcKey) + // c.log.Error(err, fmt.Sprintf("[AddPVCToLVG] an error occured while trying to add PVC %s to the cache", pvcKey)) + // return err + // } + // + // if pvcCh.selectedNode != "" && + // pvcCh.selectedNode != pvc.Annotations[SelectedNodeAnnotation] { + // c.log.Warning(fmt.Sprintf("[AddPVCToLVG] PVC %s in the cache has selected node %s, but selected node in the annotations is %s. Cached PVC selected nodename will be updated", pvcKey, pvcCh.selectedNode, pvc.Annotations[SelectedNodeAnnotation])) + // targetLVGName := c.FindLVGForPVCBySelectedNode(pvc, pvc.Annotations[SelectedNodeAnnotation]) + // if targetLVGName != "" { + // c.UpdatePVC(targetLVGName, pvc) + // } + // } + // } + //} + + return nil } pvcKey := configurePVCKey(pvc) - if c.lvgs[lvgName].pvcs[pvcKey] != nil { - return fmt.Errorf("PVC %s already exist in the cache", pvc.Name) + lvgCh, found := c.lvgs.Load(lvgName) + if !found { + return fmt.Errorf("the LVMVolumeGroup %s was not found in the cache", lvgName) } - c.lvgs[lvgName].pvcs[pvcKey] = &pvcCache{pvc: pvc, nodeName: ""} - c.pvcLVGs[pvcKey] = append(c.pvcLVGs[pvcKey], lvgName) + if pvcCh, found := lvgCh.(*lvgCache).pvcs[pvcKey]; found { + c.log.Debug(fmt.Sprintf("[AddPVCToLVG] PVC %s has been already added to the cache for the LVMVolumeGroup %s. It will be updated", pvcKey, lvgName)) - return nil -} + if pvcCh == nil { + err := fmt.Errorf("cache is not initialized for PVC %s", pvcKey) + c.log.Error(err, fmt.Sprintf("[AddPVCToLVG] an error occured while trying to add PVC %s to the cache", pvcKey)) + return err + } -func (c *Cache) UpdatePVC(lvgName string, pvc *v1.PersistentVolumeClaim) error { - pvcKey := configurePVCKey(pvc) - if c.lvgs[lvgName].pvcs[pvcKey] == nil { - return fmt.Errorf("PVC %s not found", pvc.Name) + pvcCh.pvc = pvc + return nil } - c.lvgs[lvgName].pvcs[pvcKey].pvc = pvc - c.lvgs[lvgName].pvcs[pvcKey].nodeName = pvc.Annotations[SelectedNodeAnnotation] + c.log.Debug(fmt.Sprintf("new cache will be initialized for PVC %s in LVMVolumeGroup %s", pvcKey, lvgName)) + c.addNewPVC(lvgCh.(*lvgCache), pvc, lvgName) + //lvgCh.(*lvgCache).pvcs[pvcKey] = &pvcCache{pvc: pvc, selectedNode: ""} + // + //lvgsForPVC, _ := c.pvcLVGs.Load(pvcKey) + //if lvgsForPVC == nil { + // lvgsForPVC = make([]string, 0, 5) + //} + //lvgsForPVC = append(lvgsForPVC.([]string), lvgName) + //c.pvcLVGs.Store(pvcKey, lvgsForPVC) return nil } -func (c *Cache) CheckPVCInQueue(pvc *v1.PersistentVolumeClaim) bool { +func (c *Cache) addNewPVC(lvgCh *lvgCache, pvc *v1.PersistentVolumeClaim, lvgName string) { pvcKey := configurePVCKey(pvc) + lvgCh.pvcs[pvcKey] = &pvcCache{pvc: pvc, selectedNode: pvc.Annotations[SelectedNodeAnnotation]} - if _, exist := c.pvcQueue[pvcKey]; !exist { - return false + lvgsForPVC, found := c.pvcLVGs.Load(pvcKey) + if !found || lvgsForPVC == nil { + lvgsForPVC = make([]string, 0) } - - return true + c.log.Trace(fmt.Sprintf("[addNewPVC] LVMVolumeGroups from the cache for PVC %s before append: %v", pvcKey, lvgsForPVC)) + lvgsForPVC = append(lvgsForPVC.([]string), lvgName) + c.log.Trace(fmt.Sprintf("[addNewPVC] LVMVolumeGroups from the cache for PVC %s after append: %v", pvcKey, lvgsForPVC)) + c.pvcLVGs.Store(pvcKey, lvgsForPVC) } -func (c *Cache) GetPVCsFromQueue(namespace string) map[string]*v1.PersistentVolumeClaim { - result := make(map[string]*v1.PersistentVolumeClaim, len(c.pvcQueue)) +func (c *Cache) UpdatePVC(lvgName string, pvc *v1.PersistentVolumeClaim) error { + pvcKey := configurePVCKey(pvc) - for _, pvc := range c.pvcQueue { - if pvc.Namespace == namespace { - result[pvc.Name] = pvc - } + lvgCh, found := c.lvgs.Load(lvgName) + if !found { + return fmt.Errorf("the LVMVolumeGroup %s was not found in the cache", lvgName) + } + + if lvgCh.(*lvgCache).pvcs[pvcKey] == nil { + c.log.Warning(fmt.Sprintf("[UpdatePVC] PVC %s was not found in the cache for the LVMVolumeGroup %s. It will be added", pvcKey, lvgName)) + c.AddPVCToLVG(lvgName, pvc) + return nil } - return result + lvgCh.(*lvgCache).pvcs[pvcKey].pvc = pvc + lvgCh.(*lvgCache).pvcs[pvcKey].selectedNode = pvc.Annotations[SelectedNodeAnnotation] + c.log.Debug(fmt.Sprintf("[UpdatePVC] successfully updated PVC %s with selected node %s in the cache for LVMVolumeGroup %s", pvcKey, pvc.Annotations[SelectedNodeAnnotation], lvgName)) + + return nil } -func (c *Cache) GetAllPVCByLVG(lvgName string) []*v1.PersistentVolumeClaim { - pvcsCache := c.lvgs[lvgName] +func (c *Cache) GetAllPVCForLVG(lvgName string) ([]*v1.PersistentVolumeClaim, error) { + lvgCh, found := c.lvgs.Load(lvgName) + if !found { + err := fmt.Errorf("cache was not found for the LVMVolumeGroup %s", lvgName) + c.log.Error(err, fmt.Sprintf("[GetAllPVCForLVG] an error occured while trying to get all PVC for the LVMVolumeGroup %s", lvgName)) + return nil, err + } - result := make([]*v1.PersistentVolumeClaim, 0, len(pvcsCache.pvcs)) - for _, cache := range pvcsCache.pvcs { - result = append(result, cache.pvc) + result := make([]*v1.PersistentVolumeClaim, 0, len(lvgCh.(*lvgCache).pvcs)) + for _, pvcCh := range lvgCh.(*lvgCache).pvcs { + result = append(result, pvcCh.pvc) } - return result + return result, nil } -func (c *Cache) GetPVCNodeName(lvgName string, pvc *v1.PersistentVolumeClaim) string { +func (c *Cache) GetPVCSelectedNodeName(lvgName string, pvc *v1.PersistentVolumeClaim) (string, error) { pvcKey := configurePVCKey(pvc) - return c.lvgs[lvgName].pvcs[pvcKey].nodeName + lvgCh, found := c.lvgs.Load(lvgName) + if !found { + err := fmt.Errorf("cache was not found for the LVMVolumeGroup %s", lvgName) + c.log.Error(err, fmt.Sprintf("[GetPVCSelectedNodeName] an error occured while trying to get selected node name for PVC %s in the LVMVolumeGroup %s", pvcKey, lvgName)) + return "", err + } + + if lvgCh.(*lvgCache).pvcs[pvcKey] == nil { + err := fmt.Errorf("cache was not found for PVC %s", pvcKey) + c.log.Error(err, fmt.Sprintf("[GetPVCSelectedNodeName] an error occured while trying to get selected node name for the PVC %s in the LVMVolumeGroup %s", pvcKey, lvgName)) + return "", err + } + + return lvgCh.(*lvgCache).pvcs[pvcKey].selectedNode, nil } func (c *Cache) GetLVGNamesForPVC(pvc *v1.PersistentVolumeClaim) []string { pvcKey := configurePVCKey(pvc) - return c.pvcLVGs[pvcKey] + lvgNames, found := c.pvcLVGs.Load(pvcKey) + if !found { + c.log.Warning(fmt.Sprintf("[GetLVGNamesForPVC] no cached LVMVolumeGroups were found for PVC %s", pvcKey)) + return nil + } + + return lvgNames.([]string) } func (c *Cache) RemoveBoundedPVCSpaceReservation(lvgName string, pvc *v1.PersistentVolumeClaim) error { pvcKey := configurePVCKey(pvc) - pvcCh := c.lvgs[lvgName].pvcs[pvcKey] - if pvcCh.nodeName == "" { - return fmt.Errorf("no node selected for PVC %s", pvc.Name) + + lvgCh, found := c.lvgs.Load(lvgName) + if !found { + err := fmt.Errorf("LVMVolumeGroup %s was not found in the cache", lvgName) + c.log.Error(err, fmt.Sprintf("[RemoveBoundedPVCSpaceReservation] an error occured while trying to remove space reservation for PVC %s in the LVMVolumeGroup %s", pvcKey, lvgName)) + return err } - delete(c.lvgs[lvgName].pvcs, pvcKey) - delete(c.pvcLVGs, pvcKey) - delete(c.pvcQueue, pvcKey) + pvcCh, found := lvgCh.(*lvgCache).pvcs[pvcKey] + if !found || pvcCh == nil { + err := fmt.Errorf("cache for PVC %s was not found", pvcKey) + c.log.Error(err, fmt.Sprintf("[RemoveBoundedPVCSpaceReservation] an error occured while trying to remove space reservation for PVC %s in the LVMVolumeGroup %s", pvcKey, lvgName)) + return err + } + + delete(lvgCh.(*lvgCache).pvcs, pvcKey) + c.pvcLVGs.Delete(pvcKey) return nil } -func (c *Cache) RemoveUnboundedPVCSpaceReservation(log logger.Logger, pvc *v1.PersistentVolumeClaim) error { +func (c *Cache) RemoveSpaceReservationForPVCWithSelectedNode(pvc *v1.PersistentVolumeClaim) error { pvcKey := configurePVCKey(pvc) + selectedLVGName := "" - for _, lvgCh := range c.lvgs { - pvcCh, exist := lvgCh.pvcs[pvcKey] - if exist { - if pvcCh.nodeName == "" { - delete(lvgCh.pvcs, pvcKey) - delete(c.pvcQueue, pvcKey) - log.Debug(fmt.Sprintf("[RemoveUnboundedPVCSpaceReservation] removed unbound cache PVC %s from LVG %s", pvc.Name, lvgCh.lvg.Name)) - continue - } + lvgArray, found := c.pvcLVGs.Load(pvcKey) + if !found { + c.log.Debug(fmt.Sprintf("[RemoveSpaceReservationForPVCWithSelectedNode] cache for PVC %s has been already removed", pvcKey)) + return nil + } - log.Debug(fmt.Sprintf("[RemoveUnboundedPVCSpaceReservation] PVC %s has bounded to node %s. It should not be revomed from LVG %s", pvc.Name, pvcCh.nodeName, lvgCh.lvg.Name)) + for _, lvgName := range lvgArray.([]string) { + //// this check is needed in case some LVMVolumeGroup was removed from PVC lvgArray due to wipe PVC reservation space + //if lvgName == "" { + // continue + //} + + lvgCh, found := c.lvgs.Load(lvgName) + if !found || lvgCh == nil { + err := fmt.Errorf("no cache found for the LVMVolumeGroup %s", lvgName) + c.log.Error(err, fmt.Sprintf("[RemoveSpaceReservationForPVCWithSelectedNode] an error occured while trying to remove space reservation for PVC %s", pvcKey)) + return err + } + + if _, found := lvgCh.(*lvgCache).pvcs[pvcKey]; !found { + c.log.Debug(fmt.Sprintf("[RemoveSpaceReservationForPVCWithSelectedNode] PVC %s space reservation in the LVMVolumeGroup %s has been already removed", pvcKey, lvgName)) + continue + } + + selectedNode := lvgCh.(*lvgCache).pvcs[pvcKey].selectedNode + if selectedNode == "" { + delete(lvgCh.(*lvgCache).pvcs, pvcKey) + c.log.Debug(fmt.Sprintf("[RemoveSpaceReservationForPVCWithSelectedNode] removed space reservation for PVC %s in the LVMVolumeGroup %s due the PVC got selected to the node %s", pvcKey, lvgName, selectedNode)) + } else { + selectedLVGName = lvgName + c.log.Debug(fmt.Sprintf("[RemoveSpaceReservationForPVCWithSelectedNode] PVC %s got selected to the node %s. It should not be revomed from the LVMVolumeGroup %s", pvcKey, selectedNode, lvgName)) + } + } + c.log.Debug(fmt.Sprintf("[RemoveSpaceReservationForPVCWithSelectedNode] PVC %s space reservation has been removed from LVMVolumeGroup cache", pvcKey)) + + c.log.Debug(fmt.Sprintf("[RemoveSpaceReservationForPVCWithSelectedNode] cache for PVC %s will be wiped from unused LVMVolumeGroups", pvcKey)) + cleared := make([]string, 0, len(lvgArray.([]string))) + for _, lvgName := range lvgArray.([]string) { + if lvgName == selectedLVGName { + c.log.Debug(fmt.Sprintf("[RemoveSpaceReservationForPVCWithSelectedNode] the LVMVolumeGroup %s will be saved for PVC %s cache as used", lvgName, pvcKey)) + cleared = append(cleared, lvgName) + } else { + c.log.Debug(fmt.Sprintf("[RemoveSpaceReservationForPVCWithSelectedNode] the LVMVolumeGroup %s will be removed from PVC %s cache as not used", lvgName, pvcKey)) } } + c.log.Trace(fmt.Sprintf("[RemoveSpaceReservationForPVCWithSelectedNode] cleared LVMVolumeGroups for PVC %s: %v", pvcKey, cleared)) + c.pvcLVGs.Store(pvcKey, cleared) return nil } func (c *Cache) RemovePVCSpaceReservationForced(pvc *v1.PersistentVolumeClaim) { + targetPvcKey := configurePVCKey(pvc) + + c.log.Debug(fmt.Sprintf("[RemovePVCSpaceReservationForced] run full cache wipe for PVC %s", targetPvcKey)) + c.pvcLVGs.Range(func(pvcKey, lvgArray any) bool { + if pvcKey == targetPvcKey { + for _, lvgName := range lvgArray.([]string) { + lvgCh, found := c.lvgs.Load(lvgName) + if found { + delete(lvgCh.(*lvgCache).pvcs, pvcKey.(string)) + } + } + } + + return true + }) + + c.pvcLVGs.Delete(targetPvcKey) +} + +func (c *Cache) FindLVGForPVCBySelectedNode(pvc *v1.PersistentVolumeClaim, nodeName string) string { pvcKey := configurePVCKey(pvc) - for _, lvgName := range c.pvcLVGs[pvcKey] { - delete(c.lvgs[lvgName].pvcs, pvcKey) + lvgsForPVC, found := c.pvcLVGs.Load(pvcKey) + if !found { + c.log.Debug(fmt.Sprintf("[FindLVGForPVCBySelectedNode] no LVMVolumeGroups were found in the cache for PVC %s. Returns empty string", pvcKey)) + return "" + } + + lvgsOnTheNode, found := c.nodeLVGs.Load(nodeName) + if !found { + c.log.Debug(fmt.Sprintf("[FindLVGForPVCBySelectedNode] no LVMVolumeGroups were found in the cache for the node %s. Returns empty string", nodeName)) + return "" + } + + var targetLVG string + for _, lvgName := range lvgsForPVC.([]string) { + if slices2.Contains(lvgsOnTheNode.([]string), lvgName) { + targetLVG = lvgName + } } - delete(c.pvcLVGs, pvcKey) - delete(c.pvcQueue, pvcKey) + if targetLVG == "" { + c.log.Debug(fmt.Sprintf("[FindLVGForPVCBySelectedNode] no LVMVolumeGroup was found for PVC %s. Returns empty string", pvcKey)) + } + + return targetLVG +} + +func (c *Cache) PrintTheCacheTraceLog() { + c.log.Trace("*******************CACHE BEGIN*******************") + c.log.Trace("[LVMVolumeGroups]") + c.lvgs.Range(func(lvgName, lvgCh any) bool { + c.log.Trace(fmt.Sprintf("[LVMVolumeGroup: %s]", lvgName)) + + for pvcName, pvcCh := range lvgCh.(*lvgCache).pvcs { + c.log.Trace(fmt.Sprintf(" PVC %s, selected node: %s", pvcName, pvcCh.selectedNode)) + } + + return true + }) + + c.log.Trace("") + c.log.Trace("[PVC and LVG]") + c.pvcLVGs.Range(func(pvcName, lvgs any) bool { + c.log.Trace(fmt.Sprintf("[PVC: %s]", pvcName)) + + for _, lvgName := range lvgs.([]string) { + c.log.Trace(fmt.Sprintf(" LVMVolumeGroup: %s", lvgName)) + } + + return true + }) + + c.log.Trace("") + c.log.Trace("[Node and LVG]") + c.nodeLVGs.Range(func(nodeName, lvgs any) bool { + c.log.Trace(fmt.Sprintf("[Node: %s]", nodeName)) + + for _, lvgName := range lvgs.([]string) { + c.log.Trace(fmt.Sprintf(" LVMVolumeGroup name: %s", lvgName)) + } + + return true + }) + c.log.Trace("*******************CACHE END*******************") } func configurePVCKey(pvc *v1.PersistentVolumeClaim) string { diff --git a/images/sds-local-volume-scheduler-extender/pkg/controller/lvg_watcher_cache.go b/images/sds-local-volume-scheduler-extender/pkg/controller/lvg_watcher_cache.go index ca10a229..ad42e2ab 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/controller/lvg_watcher_cache.go +++ b/images/sds-local-volume-scheduler-extender/pkg/controller/lvg_watcher_cache.go @@ -67,16 +67,15 @@ func RunLVGWatcherCacheController( log.Info(fmt.Sprintf("[RunLVGWatcherCacheController] cache was updated for the LVMVolumeGroup %s", lvg.Name)) } else { log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] the LVMVolumeGroup %s was not found. It will be added to the cache", lvg.Name)) - err = cache.AddLVG(lvg) - if err != nil { - log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to add the LVMVolumeGroup %s to the cache", lvg.Name)) - } - + cache.AddLVG(lvg) log.Info(fmt.Sprintf("[RunLVGWatcherCacheController] cache was added for the LVMVolumeGroup %s", lvg.Name)) } log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] starts to clear the cache for the LVMVolumeGroup %s", lvg.Name)) - pvcs := cache.GetAllPVCByLVG(lvg.Name) + pvcs, err := cache.GetAllPVCForLVG(lvg.Name) + if err != nil { + log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to get all PVC for the LVMVolumeGroup %s", lvg.Name)) + } for _, pvc := range pvcs { log.Trace(fmt.Sprintf("[RunLVGWatcherCacheController] cached PVC %s belongs to LVMVolumeGroup %s", pvc.Name, lvg.Name)) if pvc.Status.Phase == v1.ClaimBound { @@ -139,12 +138,15 @@ func RunLVGWatcherCacheController( } log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] successfully updated the LVMVolumeGroup %s in the cache", newLvg.Name)) - cachedPvcs := cache.GetAllPVCByLVG(newLvg.Name) - for _, pvc := range cachedPvcs { - log.Trace(fmt.Sprintf("[RunLVGWatcherCacheController] PVC %s from the cache belongs to LVMVolumeGroup %s", pvc.Name, newLvg.Name)) - log.Trace(fmt.Sprintf("[RunLVGWatcherCacheController] PVC %s has status phase %s", pvc.Name, pvc.Status.Phase)) + cachedPVCs, err := cache.GetAllPVCForLVG(newLvg.Name) + if err != nil { + log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to get all PVC for the LVMVolumeGroup %s", newLvg.Name)) + } + for _, pvc := range cachedPVCs { + log.Trace(fmt.Sprintf("[RunLVGWatcherCacheController] PVC %s/%s from the cache belongs to LVMVolumeGroup %s", pvc.Namespace, pvc.Name, newLvg.Name)) + log.Trace(fmt.Sprintf("[RunLVGWatcherCacheController] PVC %s/%s has status phase %s", pvc.Namespace, pvc.Name, pvc.Status.Phase)) if pvc.Status.Phase == v1.ClaimBound { - log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] PVC %s from the cache has Status.Phase Bound. It will be removed from the reserved space in the LVMVolumeGroup %s", pvc.Name, newLvg.Name)) + log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] PVC %s/%s from the cache has Status.Phase Bound. It will be removed from the reserved space in the LVMVolumeGroup %s", pvc.Namespace, pvc.Name, newLvg.Name)) err = cache.RemoveBoundedPVCSpaceReservation(newLvg.Name, pvc) if err != nil { log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to remove PVC %s from the cache in the LVMVolumeGroup %s", pvc.Name, newLvg.Name)) diff --git a/images/sds-local-volume-scheduler-extender/pkg/controller/pvc_watcher_cache.go b/images/sds-local-volume-scheduler-extender/pkg/controller/pvc_watcher_cache.go index e06df377..5df84d68 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/controller/pvc_watcher_cache.go +++ b/images/sds-local-volume-scheduler-extender/pkg/controller/pvc_watcher_cache.go @@ -5,11 +5,19 @@ import ( "errors" "fmt" v1 "k8s.io/api/core/v1" - cache2 "k8s.io/client-go/tools/cache" + v12 "k8s.io/api/storage/v1" + "k8s.io/client-go/util/workqueue" "k8s.io/utils/strings/slices" "sds-local-volume-scheduler-extender/pkg/cache" "sds-local-volume-scheduler-extender/pkg/logger" + "sds-local-volume-scheduler-extender/pkg/scheduler" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" ) const ( @@ -17,107 +25,152 @@ const ( ) func RunPVCWatcherCacheController( - ctx context.Context, mgr manager.Manager, log logger.Logger, schedulerCache *cache.Cache, ) error { log.Info("[RunPVCWatcherCacheController] starts the work") - inf, err := mgr.GetCache().GetInformer(ctx, &v1.PersistentVolumeClaim{}) + c, err := controller.New("test-pvc-watcher", mgr, controller.Options{ + Reconciler: reconcile.Func(func(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + return reconcile.Result{}, nil + }), + }) if err != nil { - log.Error(err, "[RunPVCWatcherCacheController] unable to get the informer") + log.Error(err, "[RunPVCWatcherCacheController] unable to create controller") return err } - _, err = inf.AddEventHandler(cache2.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - log.Info("[RunPVCWatcherCacheController] Add Func reconciliation starts") - pvc, ok := obj.(*v1.PersistentVolumeClaim) + err = c.Watch(source.Kind(mgr.GetCache(), &v1.PersistentVolumeClaim{}), handler.Funcs{ + CreateFunc: func(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) { + log.Info("[RunPVCWatcherCacheController] CreateFunc reconciliation starts") + pvc, ok := e.Object.(*v1.PersistentVolumeClaim) if !ok { err = errors.New("unable to cast event object to a given type") log.Error(err, "[RunPVCWatcherCacheController] an error occurred while handling create event") } - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] AddFunc starts the reconciliation for the PVC %s", pvc.Name)) - - switch shouldAddPVCToCache(schedulerCache, pvc) { - case true: - // Добавляем в queue, иначе фильтр не сможет получить ее из кеша - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s should be added to the cache", pvc.Name)) - schedulerCache.AddPVCToCacheQueue(pvc) - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s was added to the cache queue", pvc.Name)) - case false: - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s was found in the cache queue", pvc.Name)) - selectedNodeName, wasSelected := pvc.Annotations[cache.SelectedNodeAnnotation] - if !wasSelected { - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s should not be reconciled by Add Func due it has not selected node annotation", pvc.Name)) - return - } - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s has selected node annotation, it will be reconciled in Add func", pvc.Name)) + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] CreateFunc starts the reconciliation for the PVC %s/%s", pvc.Namespace, pvc.Name)) - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] starts to find common LVMVolumeGroup for the selected node %s and PVC %s", selectedNodeName, pvc.Name)) - log.Trace(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s has been selected to the node %s", pvc.Name, selectedNodeName)) - lvgsOnTheNode := schedulerCache.GetLVGNamesByNodeName(selectedNodeName) - for _, lvgName := range lvgsOnTheNode { - log.Trace(fmt.Sprintf("[RunPVCWatcherCacheController] LVMVolumeGroup %s belongs to the node %s", lvgName, selectedNodeName)) - } - lvgsForPVC := schedulerCache.GetLVGNamesForPVC(pvc) - for _, lvgName := range lvgsForPVC { - log.Trace(fmt.Sprintf("[RunPVCWatcherCacheController] LVMVolumeGroup %s belongs to PVC %s", lvgName, pvc.Name)) - } + selectedNodeName, wasSelected := pvc.Annotations[cache.SelectedNodeAnnotation] + if !wasSelected || + pvc.Status.Phase == v1.ClaimBound || + pvc.DeletionTimestamp != nil { + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s/%s should not be reconciled by CreateFunc", pvc.Namespace, pvc.Name)) + return + } + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s/%s has selected node annotation, it will be reconciled in CreateFunc", pvc.Namespace, pvc.Name)) + log.Trace(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s/%s has been selected to the node %s", pvc.Namespace, pvc.Name, selectedNodeName)) - var commonLVGName string - for _, pvcLvg := range lvgsForPVC { - if slices.Contains(lvgsOnTheNode, pvcLvg) { - commonLVGName = pvcLvg - } - } - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] successfully found common LVMVolumeGroup %s for the selected node %s and PVC %s", commonLVGName, selectedNodeName, pvc.Name)) + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] starts to find common LVMVolumeGroup for the selected node %s and PVC %s/%s", selectedNodeName, pvc.Namespace, pvc.Name)) + lvgsOnTheNode := schedulerCache.GetLVGNamesByNodeName(selectedNodeName) + for _, lvgName := range lvgsOnTheNode { + log.Trace(fmt.Sprintf("[RunPVCWatcherCacheController] LVMVolumeGroup %s belongs to the node %s", lvgName, selectedNodeName)) + } - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] starts to update PVC %s in the cache", pvc.Name)) - log.Trace(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s has status phase: %s", pvc.Name, pvc.Status.Phase)) - err = schedulerCache.UpdatePVC(commonLVGName, pvc) + lvgsForPVC := schedulerCache.GetLVGNamesForPVC(pvc) + if lvgsForPVC == nil || len(lvgsForPVC) == 0 { + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] no LVMVolumeGroups were found in the cache for PVC %s/%s. Use Storage Class %s instead", pvc.Namespace, pvc.Name, *pvc.Spec.StorageClassName)) + sc := &v12.StorageClass{} + err = mgr.GetClient().Get(ctx, client.ObjectKey{ + Name: *pvc.Spec.StorageClassName, + }, sc) if err != nil { - log.Error(err, fmt.Sprintf("[RunPVCWatcherCacheController] unable to update PVC %s in the cache", pvc.Name)) + log.Error(err, fmt.Sprintf("[RunPVCWatcherCacheController] unable to get Storage Class %s for PVC %s/%s", *pvc.Spec.StorageClassName, pvc.Namespace, pvc.Name)) return } - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] successfully updated PVC %s in the cache", pvc.Name)) - // У PVC выбралась нода, но она еще не в баунд (в кеше PVC без ноды на лвгхах) - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] starts to remove unbound PVC %s space reservation from the cache", pvc.Name)) - err = schedulerCache.RemoveUnboundedPVCSpaceReservation(log, pvc) + lvgsFromSc, err := scheduler.ExtractLVGsFromSC(sc) if err != nil { - log.Error(err, fmt.Sprintf("[RunPVCWatcherCacheController] unable to remove space reservation in the cache for unbounded PVC %s", pvc.Name)) + log.Error(err, fmt.Sprintf("[RunPVCWatcherCacheController] unable to extract LVMVolumeGroups from the Storage Class %s", sc.Name)) + } + + for _, lvg := range lvgsFromSc { + lvgsForPVC = append(lvgsForPVC, lvg.Name) } - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] succefully ended the unbound PVC %s space reservation from the cache", pvc.Name)) } + for _, lvgName := range lvgsForPVC { + log.Trace(fmt.Sprintf("[RunPVCWatcherCacheController] LVMVolumeGroup %s belongs to PVC %s/%s", lvgName, pvc.Namespace, pvc.Name)) + } + + var commonLVGName string + for _, pvcLvg := range lvgsForPVC { + if slices.Contains(lvgsOnTheNode, pvcLvg) { + commonLVGName = pvcLvg + } + } + if commonLVGName == "" { + log.Error(errors.New("common LVMVolumeGroup was not found"), fmt.Sprintf("[RunPVCWatcherCacheController] unable to identify a LVMVolumeGroup for PVC %s/%s", pvc.Namespace, pvc.Name)) + return + } + + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] successfully found common LVMVolumeGroup %s for the selected node %s and PVC %s/%s", commonLVGName, selectedNodeName, pvc.Namespace, pvc.Name)) + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] starts to update PVC %s/%s in the cache", pvc.Namespace, pvc.Name)) + log.Trace(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s/%s has status phase: %s", pvc.Namespace, pvc.Name, pvc.Status.Phase)) + err = schedulerCache.UpdatePVC(commonLVGName, pvc) + if err != nil { + log.Error(err, fmt.Sprintf("[RunPVCWatcherCacheController] unable to update PVC %s/%s in the cache", pvc.Namespace, pvc.Name)) + return + } + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] successfully updated PVC %s/%s in the cache", pvc.Namespace, pvc.Name)) + + schedulerCache.PrintTheCacheTraceLog() + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] starts to remove space reservation for PVC %s/%s with selected node from the cache", pvc.Namespace, pvc.Name)) + err = schedulerCache.RemoveSpaceReservationForPVCWithSelectedNode(pvc) + if err != nil { + log.Error(err, fmt.Sprintf("[RunPVCWatcherCacheController] unable to remove PVC %s/%s space reservation in the cache", pvc.Namespace, pvc.Name)) + return + } + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] successfully removed space reservation for PVC %s/%s with selected node", pvc.Namespace, pvc.Name)) + schedulerCache.PrintTheCacheTraceLog() + + log.Info("[RunPVCWatcherCacheController] CreateFunc reconciliation ends") }, - UpdateFunc: func(oldObj, newObj interface{}) { + UpdateFunc: func(ctx context.Context, e event.UpdateEvent, q workqueue.RateLimitingInterface) { log.Info("[RunPVCWatcherCacheController] Update Func reconciliation starts") - pvc, ok := newObj.(*v1.PersistentVolumeClaim) + pvc, ok := e.ObjectNew.(*v1.PersistentVolumeClaim) if !ok { err = errors.New("unable to cast event object to a given type") log.Error(err, "[RunPVCWatcherCacheController] an error occurred while handling create event") } - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] UpdateFunc starts the reconciliation for the PVC %s", pvc.Name)) + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] UpdateFunc starts the reconciliation for the PVC %s/%s", pvc.Namespace, pvc.Name)) - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s was found in the cache queue", pvc.Name)) selectedNodeName, wasSelected := pvc.Annotations[cache.SelectedNodeAnnotation] - if !wasSelected { - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s should not be reconciled by Add Func due it has not selected node annotation", pvc.Name)) + if !wasSelected || pvc.DeletionTimestamp != nil { + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s/%s should not be reconciled by UpdateFunc", pvc.Namespace, pvc.Name)) return } - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s has selected node annotation, it will be reconciled in Add func", pvc.Name)) + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s/%s has selected node annotation, it will be reconciled in UpdateFunc", pvc.Namespace, pvc.Name)) + log.Trace(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s/%s has been selected to the node %s", pvc.Namespace, pvc.Name, selectedNodeName)) - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] starts to find common LVMVolumeGroup for the selected node %s and PVC %s", selectedNodeName, pvc.Name)) - log.Trace(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s has been selected to the node %s", pvc.Name, selectedNodeName)) + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] starts to find common LVMVolumeGroup for the selected node %s and PVC %s/%s", selectedNodeName, pvc.Namespace, pvc.Name)) lvgsOnTheNode := schedulerCache.GetLVGNamesByNodeName(selectedNodeName) for _, lvgName := range lvgsOnTheNode { log.Trace(fmt.Sprintf("[RunPVCWatcherCacheController] LVMVolumeGroup %s belongs to the node %s", lvgName, selectedNodeName)) } + lvgsForPVC := schedulerCache.GetLVGNamesForPVC(pvc) + if lvgsForPVC == nil || len(lvgsForPVC) == 0 { + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] no LVMVolumeGroups were found in the cache for PVC %s/%s. Use Storage Class %s instead", pvc.Namespace, pvc.Name, *pvc.Spec.StorageClassName)) + sc := &v12.StorageClass{} + err = mgr.GetClient().Get(ctx, client.ObjectKey{ + Name: *pvc.Spec.StorageClassName, + }, sc) + if err != nil { + log.Error(err, fmt.Sprintf("[RunPVCWatcherCacheController] unable to get Storage Class %s for PVC %s/%s", *pvc.Spec.StorageClassName, pvc.Namespace, pvc.Name)) + return + } + + lvgsFromSc, err := scheduler.ExtractLVGsFromSC(sc) + if err != nil { + log.Error(err, fmt.Sprintf("[RunPVCWatcherCacheController] unable to extract LVMVolumeGroups from the Storage Class %s", sc.Name)) + } + + for _, lvg := range lvgsFromSc { + lvgsForPVC = append(lvgsForPVC, lvg.Name) + } + } for _, lvgName := range lvgsForPVC { - log.Trace(fmt.Sprintf("[RunPVCWatcherCacheController] LVMVolumeGroup %s belongs to PVC %s", lvgName, pvc.Name)) + log.Trace(fmt.Sprintf("[RunPVCWatcherCacheController] LVMVolumeGroup %s belongs to PVC %s/%s", lvgName, pvc.Namespace, pvc.Name)) } var commonLVGName string @@ -126,27 +179,38 @@ func RunPVCWatcherCacheController( commonLVGName = pvcLvg } } - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] successfully found common LVMVolumeGroup %s for the selected node %s and PVC %s", commonLVGName, selectedNodeName, pvc.Name)) + if commonLVGName == "" { + log.Error(errors.New("common LVMVolumeGroup was not found"), fmt.Sprintf("[RunPVCWatcherCacheController] unable to identify a LVMVolumeGroup for PVC %s/%s", pvc.Namespace, pvc.Name)) + return + } - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] starts to update PVC %s in the cache", pvc.Name)) - log.Trace(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s has status phase: %s", pvc.Name, pvc.Status.Phase)) + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] successfully found common LVMVolumeGroup %s for the selected node %s and PVC %s/%s", commonLVGName, selectedNodeName, pvc.Namespace, pvc.Name)) + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] starts to update PVC %s/%s in the cache", pvc.Namespace, pvc.Name)) + log.Trace(fmt.Sprintf("[RunPVCWatcherCacheController] PVC %s/%s has status phase: %s", pvc.Namespace, pvc.Name, pvc.Status.Phase)) err = schedulerCache.UpdatePVC(commonLVGName, pvc) if err != nil { - log.Error(err, fmt.Sprintf("[RunPVCWatcherCacheController] unable to update PVC %s in the cache", pvc.Name)) + log.Error(err, fmt.Sprintf("[RunPVCWatcherCacheController] unable to update PVC %s/%s in the cache", pvc.Namespace, pvc.Name)) return } - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] successfully updated PVC %s in the cache", pvc.Name)) + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] successfully updated PVC %s/%s in the cache", pvc.Namespace, pvc.Name)) - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] starts to remove unbound PVC %s space reservation from the cache", pvc.Name)) - err = schedulerCache.RemoveUnboundedPVCSpaceReservation(log, pvc) + schedulerCache.PrintTheCacheTraceLog() + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] starts to remove space reservation for PVC %s/%s with selected node from the cache", pvc.Namespace, pvc.Name)) + err = schedulerCache.RemoveSpaceReservationForPVCWithSelectedNode(pvc) if err != nil { - log.Error(err, fmt.Sprintf("[RunPVCWatcherCacheController] unable to remove space reservation in the cache for unbounded PVC %s", pvc.Name)) + log.Error(err, fmt.Sprintf("[RunPVCWatcherCacheController] unable to remove PVC %s/%s space reservation in the cache", pvc.Namespace, pvc.Name)) + return } - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] succefully ended the unbound PVC %s space reservation from the cache", pvc.Name)) + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] successfully removed space reservation for PVC %s/%s with selected node", pvc.Namespace, pvc.Name)) + schedulerCache.PrintTheCacheTraceLog() + + //TODO: перенос удаления PVC в статусе Bound из кэша в данный контроллер вместо текущего в LVGWatcher + + log.Info("[RunPVCWatcherCacheController] Update Func reconciliation ends") }, - DeleteFunc: func(obj interface{}) { + DeleteFunc: func(ctx context.Context, e event.DeleteEvent, q workqueue.RateLimitingInterface) { log.Info("[RunPVCWatcherCacheController] Delete Func reconciliation starts") - pvc, ok := obj.(*v1.PersistentVolumeClaim) + pvc, ok := e.Object.(*v1.PersistentVolumeClaim) if !ok { err = errors.New("unable to cast event object to a given type") log.Error(err, "[RunPVCWatcherCacheController] an error occurred while handling create event") @@ -159,63 +223,9 @@ func RunPVCWatcherCacheController( }, }) if err != nil { - log.Error(err, "[RunPVCWatcherCacheController] unable to add event handler to the informer") + log.Error(err, "[RunPVCWatcherCacheController] unable to controller Watch") + return err } return nil } - -func shouldAddPVCToCache(schedulerCache *cache.Cache, pvc *v1.PersistentVolumeClaim) bool { - if pvc.Status.Phase == v1.ClaimBound { - return false - } - - exist := schedulerCache.CheckPVCInQueue(pvc) - if exist { - return false - } - - return true -} - -// -//func findLVGByPVC(ctx context.Context, cl client.Client, pvc *v1.PersistentVolumeClaim) (*v1alpha1.LvmVolumeGroup, error) { -// sc := &v12.StorageClass{} -// // TODO: Будет ли проставлен storage class для PVC, если не будет указан явно (иначе зачем тут поинтер?) -// err := cl.Get(ctx, client.ObjectKey{ -// Name: *pvc.Spec.StorageClassName, -// }, sc) -// if err != nil { -// return nil, fmt.Errorf("[findLVGByPVC] unable to get a storage class %s", *pvc.Spec.StorageClassName) -// } -// -// lvgsFromSC, err := scheduler.ExtractLVGsFromSC(sc) -// -// lvgList := &v1alpha1.LvmVolumeGroupList{} -// err = cl.List(ctx, lvgList) -// if err != nil { -// return nil, fmt.Errorf("[findLVGByPVC] unable to list LVMVolumeGroups") -// } -// -// lvgs := make(map[string]v1alpha1.LvmVolumeGroup, len(lvgList.Items)) -// for _, lvg := range lvgList.Items { -// lvgs[lvg.Name] = lvg -// } -// -// for _, lvg := range lvgsFromSC { -// kubeLVG, exist := lvgs[lvg.Name] -// if !exist { -// return nil, fmt.Errorf("unable to found the LVMVolumeGroup %s for storage class %s", lvg.Name, sc.Name) -// } -// -// if kubeLVG.Status.Nodes == nil || len(kubeLVG.Status.Nodes) == 0 { -// return nil, fmt.Errorf("no nodes specified for the LVMVolumeGroup %s for storage class %s", lvg.Name, sc.Name) -// } -// -// if kubeLVG.Status.Nodes[0].Name == pvc.Annotations[cache.SelectedNodeAnnotation] { -// return &kubeLVG, nil -// } -// } -// -// return nil, nil -//} diff --git a/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go b/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go index 1dc6a387..58bd6eae 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go +++ b/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go @@ -58,7 +58,12 @@ func (s *scheduler) filter(w http.ResponseWriter, r *http.Request) { s.log.Trace(fmt.Sprintf("[filter] a node from request, name :%s", n.Name)) } - pvcs := getUsedPVC(s.log, s.cache, input.Pod) + pvcs, err := getUsedPVC(s.ctx, s.client, s.log, input.Pod) + if err != nil { + s.log.Error(err, fmt.Sprintf("[filter] unable to get used PVC for a Pod %s in the namespace %s", input.Pod.Name, input.Pod.Namespace)) + http.Error(w, "internal server error", http.StatusInternalServerError) + return + } if len(pvcs) == 0 { s.log.Error(fmt.Errorf("no PVC was found for pod %s in namespace %s", input.Pod.Name, input.Pod.Namespace), fmt.Sprintf("[filter] unable to get used PVC for Pod %s", input.Pod.Name)) http.Error(w, "bad request", http.StatusBadRequest) @@ -66,7 +71,7 @@ func (s *scheduler) filter(w http.ResponseWriter, r *http.Request) { } for _, pvc := range pvcs { - s.log.Trace(fmt.Sprintf("[filter] used PVC: %s", pvc.Name)) + s.log.Trace(fmt.Sprintf("[filter] Pod %s/%s used PVC: %s", input.Pod.Namespace, input.Pod.Name, pvc.Name)) } scs, err := getStorageClassesUsedByPVCs(s.ctx, s.client, pvcs) @@ -98,6 +103,7 @@ func (s *scheduler) filter(w http.ResponseWriter, r *http.Request) { s.log.Debug("[filter] successfully filtered the nodes") s.log.Debug("[filter] starts to populate the cache") + s.cache.PrintTheCacheTraceLog() err = populateCache(s.log, filteredNodes.Nodes.Items, input.Pod, s.cache, pvcs, scs) if err != nil { s.log.Error(err, "[filter] unable to populate cache") @@ -105,6 +111,7 @@ func (s *scheduler) filter(w http.ResponseWriter, r *http.Request) { return } s.log.Debug("[filter] successfully populated the cache") + s.cache.PrintTheCacheTraceLog() w.Header().Set("content-type", "application/json") err = json.NewEncoder(w).Encode(filteredNodes) @@ -118,24 +125,27 @@ func (s *scheduler) filter(w http.ResponseWriter, r *http.Request) { func populateCache(log logger.Logger, nodes []corev1.Node, pod *corev1.Pod, schedulerCache *cache.Cache, pvcs map[string]*corev1.PersistentVolumeClaim, scs map[string]*v1.StorageClass) error { for _, node := range nodes { + log.Debug(fmt.Sprintf("[populateCache] starts the work for node %s", node.Name)) for _, volume := range pod.Spec.Volumes { if volume.PersistentVolumeClaim != nil { + log.Debug(fmt.Sprintf("[populateCache] reconcile the PVC %s for Pod %s/%s", volume.PersistentVolumeClaim.ClaimName, pod.Namespace, pod.Name)) lvgNamesForTheNode := schedulerCache.GetLVGNamesByNodeName(node.Name) - log.Trace(fmt.Sprintf("[populateCache] LVGs from cache for the node %s: %v", node.Name, lvgNamesForTheNode)) + log.Trace(fmt.Sprintf("[populateCache] LVMVolumeGroups from cache for the node %s: %v", node.Name, lvgNamesForTheNode)) pvc := pvcs[volume.PersistentVolumeClaim.ClaimName] sc := scs[*pvc.Spec.StorageClassName] + if sc.Parameters[lvmTypeParamKey] == thick { log.Debug(fmt.Sprintf("[populateCache] Storage Class %s has device type Thick, so the cache will be populated by PVC space requests", sc.Name)) lvgsForPVC, err := ExtractLVGsFromSC(sc) if err != nil { return err } - log.Trace(fmt.Sprintf("[populateCache] LVGs for PVC %s: %+v", volume.PersistentVolumeClaim.ClaimName, lvgsForPVC)) + log.Trace(fmt.Sprintf("[populateCache] LVMVolumeGroups from Storage Class %s for PVC %s: %+v", sc.Name, volume.PersistentVolumeClaim.ClaimName, lvgsForPVC)) for _, lvg := range lvgsForPVC { if slices.Contains(lvgNamesForTheNode, lvg.Name) { - log.Trace(fmt.Sprintf("[populateCache] LVG %s in the cache will be populated with unbounded PVC %s", lvg.Name, volume.PersistentVolumeClaim.ClaimName)) - err = schedulerCache.AddUnboundedPVCToLVG(lvg.Name, pvcs[volume.PersistentVolumeClaim.ClaimName]) + log.Trace(fmt.Sprintf("[populateCache] PVC %s will reserve space in LVMVolumeGroup %s cache", volume.PersistentVolumeClaim.ClaimName, lvg.Name)) + err = schedulerCache.AddPVCToLVG(lvg.Name, pvcs[volume.PersistentVolumeClaim.ClaimName]) if err != nil { return err } @@ -230,22 +240,6 @@ func filterNodes( log.Trace(fmt.Sprintf("[filterNodes] LVMVolumeGroup %s in the cache", lvg.Name)) } - lvgsThickFree, err := getLVGThickFreeSpaces(lvgs) - if err != nil { - return nil, err - } - log.Trace(fmt.Sprintf("[filterNodes] current LVGs Thick FreeSpace on the node: %+v", lvgsThickFree)) - - for lvgName, freeSpace := range lvgsThickFree { - log.Trace(fmt.Sprintf("[filterNodes] current LVG %s Thick free space %s", lvgName, resource.NewQuantity(freeSpace, resource.BinarySI))) - reservedSize := schedulerCache.GetLVGReservedSpace(lvgName) - log.Trace(fmt.Sprintf("[filterNodes] current LVG %s reserved PVC space %s", lvgName, resource.NewQuantity(reservedSize, resource.BinarySI))) - lvgsThickFree[lvgName] -= reservedSize - } - log.Trace(fmt.Sprintf("[filterNodes] current LVGs Thick FreeSpace with reserved PVC: %+v", lvgsThickFree)) - - lvgsThickFreeMutex := &sync.RWMutex{} - log.Debug("[filterNodes] starts to get LVMVolumeGroups for Storage Classes") scLVGs, err := GetSortedLVGsFromStorageClasses(scs) if err != nil { @@ -262,6 +256,25 @@ func filterNodes( for _, lvg := range usedLVGs { log.Trace(fmt.Sprintf("[filterNodes] the LVMVolumeGroup %s is actually used", lvg.Name)) } + lvgsThickFree, err := getLVGThickFreeSpaces(usedLVGs) + if err != nil { + return nil, err + } + log.Trace(fmt.Sprintf("[filterNodes] current LVMVolumeGroups Thick FreeSpace on the node: %+v", lvgsThickFree)) + + for lvgName, freeSpace := range lvgsThickFree { + log.Trace(fmt.Sprintf("[filterNodes] current LVMVolumeGroup %s Thick free space %s", lvgName, resource.NewQuantity(freeSpace, resource.BinarySI))) + reservedSize, err := schedulerCache.GetLVGReservedSpace(lvgName) + if err != nil { + log.Error(err, fmt.Sprintf("[filterNodes] unable to cound cache reserved size for the LVMVolumeGroup %s", lvgName)) + continue + } + log.Trace(fmt.Sprintf("[filterNodes] current LVMVolumeGroup %s reserved PVC space %s", lvgName, resource.NewQuantity(reservedSize, resource.BinarySI))) + lvgsThickFree[lvgName] -= reservedSize + } + log.Trace(fmt.Sprintf("[filterNodes] current LVMVolumeGroups Thick FreeSpace with reserved PVC: %+v", lvgsThickFree)) + + lvgsThickFreeMutex := &sync.RWMutex{} nodeLVGs := SortLVGsByNodeName(usedLVGs) for n, ls := range nodeLVGs { @@ -533,22 +546,6 @@ func SortLVGsByNodeName(lvgs map[string]*v1alpha1.LvmVolumeGroup) map[string][]* return sorted } -// TODO: remove it if no need -func GetLVMVolumeGroups(ctx context.Context, cl client.Client) (map[string]v1alpha1.LvmVolumeGroup, error) { - lvgl := &v1alpha1.LvmVolumeGroupList{} - err := cl.List(ctx, lvgl) - if err != nil { - return nil, err - } - - lvgMap := make(map[string]v1alpha1.LvmVolumeGroup, len(lvgl.Items)) - for _, lvg := range lvgl.Items { - lvgMap[lvg.Name] = lvg - } - - return lvgMap, nil -} - func getVGFreeSpace(lvg *v1alpha1.LvmVolumeGroup) (resource.Quantity, error) { free, err := resource.ParseQuantity(lvg.Status.VGSize) if err != nil { @@ -618,14 +615,19 @@ func getStorageClassesUsedByPVCs(ctx context.Context, cl client.Client, pvcs map return result, nil } -func getUsedPVC(log logger.Logger, schedulerCache *cache.Cache, pod *corev1.Pod) map[string]*corev1.PersistentVolumeClaim { +func getUsedPVC(ctx context.Context, cl client.Client, log logger.Logger, pod *corev1.Pod) (map[string]*corev1.PersistentVolumeClaim, error) { usedPvc := make(map[string]*corev1.PersistentVolumeClaim, len(pod.Spec.Volumes)) - //TODO: может сделать так, чтобы в случае, если мы не можем получить из кеша, идти в куб? или будем честно "падать"? + var err error for { - pvcMap := schedulerCache.GetPVCsFromQueue(pod.Namespace) - for _, pvc := range pvcMap { - log.Trace(fmt.Sprintf("[getUsedPVC] PVC %s from namespace %s was found in the cache", pvc.Name, pod.Namespace)) + pvcMap, err := getAllPVCsFromNamespace(ctx, cl, pod.Namespace) + if err != nil { + log.Error(err, fmt.Sprintf("[getUsedPVC] unable to get all PVC for Pod %s in the namespace %s", pod.Name, pod.Namespace)) + return nil, err + } + + for pvcName := range pvcMap { + log.Trace(fmt.Sprintf("[getUsedPVC] PVC %s is in namespace %s", pod.Namespace, pvcName)) } for _, volume := range pod.Spec.Volumes { @@ -638,7 +640,6 @@ func getUsedPVC(log logger.Logger, schedulerCache *cache.Cache, pod *corev1.Pod) filled := false if len(pvcMap) > 0 { filled = true - for _, volume := range pod.Spec.Volumes { if volume.PersistentVolumeClaim != nil { if _, added := usedPvc[volume.PersistentVolumeClaim.ClaimName]; !added { @@ -660,8 +661,22 @@ func getUsedPVC(log logger.Logger, schedulerCache *cache.Cache, pod *corev1.Pod) log.Debug(fmt.Sprintf("[getUsedPVC] Every PVC for Pod %s was found in the cache", pod.Name)) break } + } + + return usedPvc, err +} + +func getAllPVCsFromNamespace(ctx context.Context, cl client.Client, namespace string) (map[string]*corev1.PersistentVolumeClaim, error) { + list := &corev1.PersistentVolumeClaimList{} + err := cl.List(ctx, list, &client.ListOptions{Namespace: namespace}) + if err != nil { + return nil, err + } + pvcs := make(map[string]*corev1.PersistentVolumeClaim, len(list.Items)) + for _, pvc := range list.Items { + pvcs[pvc.Name] = &pvc } - return usedPvc + return pvcs, nil } diff --git a/images/sds-local-volume-scheduler-extender/pkg/scheduler/prioritize.go b/images/sds-local-volume-scheduler-extender/pkg/scheduler/prioritize.go index 0ad21dc5..e6459646 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/scheduler/prioritize.go +++ b/images/sds-local-volume-scheduler-extender/pkg/scheduler/prioritize.go @@ -42,9 +42,14 @@ func (s *scheduler) prioritize(w http.ResponseWriter, r *http.Request) { return } - pvcs := getUsedPVC(s.log, s.cache, input.Pod) + pvcs, err := getUsedPVC(s.ctx, s.client, s.log, input.Pod) if err != nil { s.log.Error(err, "[prioritize] unable to get PVC from the Pod") + http.Error(w, "internal server error", http.StatusInternalServerError) + return + } + if len(pvcs) == 0 { + s.log.Error(fmt.Errorf("no PVC was found for pod %s in namespace %s", input.Pod.Name, input.Pod.Namespace), fmt.Sprintf("[filter] unable to get used PVC for Pod %s", input.Pod.Name)) http.Error(w, "bad request", http.StatusBadRequest) return } @@ -130,7 +135,6 @@ func scoreNodes( lvgsFromNode := nodeLVGs[node.Name] var totalFreeSpaceLeft int64 - // TODO: change pvs to vgs for _, pvc := range pvcs { pvcReq := pvcRequests[pvc.Name] lvgsFromSC := scLVGs[*pvc.Spec.StorageClassName] @@ -152,8 +156,12 @@ func scoreNodes( return } log.Trace(fmt.Sprintf("[scoreNodes] LVMVolumeGroup %s free thick space before PVC reservation: %s", lvg.Name, freeSpace.String())) - reserved := schedulerCache.GetLVGReservedSpace(lvg.Name) - log.Trace(fmt.Sprintf("[scoreNodes] LVG %s PVC Space reservation: %s", lvg.Name, resource.NewQuantity(reserved, resource.BinarySI))) + reserved, err := schedulerCache.GetLVGReservedSpace(lvg.Name) + if err != nil { + log.Error(err, fmt.Sprintf("[scoreNodes] unable to count reserved space for the LVMVolumeGroup %s", lvg.Name)) + continue + } + log.Trace(fmt.Sprintf("[scoreNodes] LVMVolumeGroup %s PVC Space reservation: %s", lvg.Name, resource.NewQuantity(reserved, resource.BinarySI))) spaceWithReserved := freeSpace.Value() - reserved freeSpace = *resource.NewQuantity(spaceWithReserved, resource.BinarySI) log.Trace(fmt.Sprintf("[scoreNodes] LVMVolumeGroup %s free thick space after PVC reservation: %s", lvg.Name, freeSpace.String())) diff --git a/images/sds-local-volume-scheduler-extender/pkg/scheduler/route.go b/images/sds-local-volume-scheduler-extender/pkg/scheduler/route.go index f7c52128..1a02e090 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/scheduler/route.go +++ b/images/sds-local-volume-scheduler-extender/pkg/scheduler/route.go @@ -78,6 +78,8 @@ func status(w http.ResponseWriter, r *http.Request) { func (s *scheduler) getCache(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) + s.cache.PrintTheCacheTraceLog() + result := make(map[string][]struct { pvcName string nodeName string @@ -86,7 +88,10 @@ func (s *scheduler) getCache(w http.ResponseWriter, r *http.Request) { lvgs := s.cache.GetAllLVG() //s.log.Info(fmt.Sprintf("LVG from cache: %v", lvgs)) for _, lvg := range lvgs { - pvcs := s.cache.GetAllPVCByLVG(lvg.Name) + pvcs, err := s.cache.GetAllPVCForLVG(lvg.Name) + if err != nil { + s.log.Error(err, "something bad") + } //for _, pvc := range pvcs { // s.log.Trace(fmt.Sprintf("LVG %s has PVC from cache: %v", lvg, pvc.Name)) //} @@ -97,10 +102,14 @@ func (s *scheduler) getCache(w http.ResponseWriter, r *http.Request) { }, 0) for _, pvc := range pvcs { + selectedNode, err := s.cache.GetPVCSelectedNodeName(lvg.Name, pvc) + if err != nil { + s.log.Error(err, "something bad") + } result[lvg.Name] = append(result[lvg.Name], struct { pvcName string nodeName string - }{pvcName: pvc.Name, nodeName: s.cache.GetPVCNodeName(lvg.Name, pvc)}) + }{pvcName: pvc.Name, nodeName: selectedNode}) } } //s.log.Info(fmt.Sprintf("Result len: %d", len(result))) diff --git a/images/webhooks/src/go.mod b/images/webhooks/src/go.mod index 4e3cc2ab..49eb09c2 100644 --- a/images/webhooks/src/go.mod +++ b/images/webhooks/src/go.mod @@ -1,6 +1,6 @@ module webhooks -go 1.21.1 +go 1.21 require ( github.com/sirupsen/logrus v1.9.3 From 5596d443938e337076c31639c3f9ae74c3c04a4a Mon Sep 17 00:00:00 2001 From: Viktor Kramarenko Date: Thu, 4 Apr 2024 13:35:01 +0300 Subject: [PATCH 14/21] little refactoring Signed-off-by: Viktor Kramarenko --- .../pkg/cache/cache.go | 101 ++++++------------ .../pkg/controller/lvg_watcher_cache.go | 12 +-- 2 files changed, 36 insertions(+), 77 deletions(-) diff --git a/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go b/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go index 7658cf48..6995efe7 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go +++ b/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go @@ -11,6 +11,7 @@ import ( const ( pvcPerLVGCount = 150 + lvgsPerPVCCount = 5 SelectedNodeAnnotation = "volume.kubernetes.io/selected-node" ) @@ -137,7 +138,6 @@ func (c *Cache) AddPVCToLVG(lvgName string, pvc *v1.PersistentVolumeClaim) error c.log.Debug(fmt.Sprintf("PVC %s/%s has selected node anotation, selected node: %s", pvc.Namespace, pvc.Name, pvc.Annotations[SelectedNodeAnnotation])) pvcKey := configurePVCKey(pvc) - // если уже выбрана нода, то нам нужно проверить, что переданная лвг принадлежит этой ноде и если так, то мы просто обновим ее в ней, если не принаджелит, то скипаем с варнингом lvgsOnTheNode, found := c.nodeLVGs.Load(pvc.Annotations[SelectedNodeAnnotation]) if !found { err := fmt.Errorf("no LVMVolumeGroups found for the node %s", pvc.Annotations[SelectedNodeAnnotation]) @@ -145,64 +145,34 @@ func (c *Cache) AddPVCToLVG(lvgName string, pvc *v1.PersistentVolumeClaim) error return err } - if slices2.Contains(lvgsOnTheNode.([]string), lvgName) { - c.log.Debug(fmt.Sprintf("[AddPVCToLVG] LVMVolumeGroup %s belongs to PVC %s/%s selected node %s", lvgName, pvc.Namespace, pvc.Name, pvc.Annotations[SelectedNodeAnnotation])) - lvgCh, found := c.lvgs.Load(lvgName) - if !found { - err := fmt.Errorf("the LVMVolumeGroup %s was not found in the cache", lvgName) - c.log.Error(err, fmt.Sprintf("[AddPVCToLVG] an error occured while trying to add PVC %s to the cache", pvcKey)) - return err - } - - pvcCh, found := lvgCh.(*lvgCache).pvcs[pvcKey] - if found { - c.log.Debug(fmt.Sprintf("[AddPVCToLVG] PVC %s cache has been already added to the LVMVolumeGroup %s. It will be updated", pvcKey, lvgName)) - pvcCh.pvc = pvc - } else { - c.log.Debug(fmt.Sprintf("[AddPVCToLVG] PVC %s cache was not found in LVMVolumeGroup %s. It will be added", pvcKey, lvgName)) - c.addNewPVC(lvgCh.(*lvgCache), pvc, lvgName) - } - } else { + if !slices2.Contains(lvgsOnTheNode.([]string), lvgName) { c.log.Debug(fmt.Sprintf("[AddPVCToLVG] LVMVolumeGroup %s does not belong to PVC %s/%s selected node %s. It will be skipped", lvgName, pvc.Namespace, pvc.Name, pvc.Annotations[SelectedNodeAnnotation])) + return nil + } + + c.log.Debug(fmt.Sprintf("[AddPVCToLVG] LVMVolumeGroup %s belongs to PVC %s/%s selected node %s", lvgName, pvc.Namespace, pvc.Name, pvc.Annotations[SelectedNodeAnnotation])) + lvgCh, found := c.lvgs.Load(lvgName) + if !found { + err := fmt.Errorf("the LVMVolumeGroup %s was not found in the cache", lvgName) + c.log.Error(err, fmt.Sprintf("[AddPVCToLVG] an error occured while trying to add PVC %s to the cache", pvcKey)) + return err } - //lvgsForPVC, found := c.pvcLVGs.Load(pvcKey) - //if !found { - // c.log.Warning(fmt.Sprintf("LVMVolumeGroups were not found in the cache for PVC %s. The selected node check will be skipped, and ", pvcKey)) - // return err - //} - // - //c.log.Trace(fmt.Sprintf("[AddPVCToLVG] PVC %s has %d LVMVolumeGroups in the cache: %s", pvcKey, len(lvgsForPVC.([]string)), strings.Join(lvgsForPVC.([]string), ","))) - //for _, pvcLvgName := range lvgsForPVC.([]string) { - // lvgCh, found := c.lvgs.Load(pvcLvgName) - // if !found { - // err := fmt.Errorf("LVMVolumeGroup %s was not found in the cache", pvcLvgName) - // c.log.Error(err, fmt.Sprintf("[AddPVCToLVG] an error occured while trying to add PVC %s to the cache", pvcKey)) - // return err - // } - // - // if pvcCh, exist := lvgCh.(*lvgCache).pvcs[pvcKey]; exist { - // if pvcCh == nil { - // err := fmt.Errorf("cache struct is not initialized for PVC %s", pvcKey) - // c.log.Error(err, fmt.Sprintf("[AddPVCToLVG] an error occured while trying to add PVC %s to the cache", pvcKey)) - // return err - // } - // - // if pvcCh.selectedNode != "" && - // pvcCh.selectedNode != pvc.Annotations[SelectedNodeAnnotation] { - // c.log.Warning(fmt.Sprintf("[AddPVCToLVG] PVC %s in the cache has selected node %s, but selected node in the annotations is %s. Cached PVC selected nodename will be updated", pvcKey, pvcCh.selectedNode, pvc.Annotations[SelectedNodeAnnotation])) - // targetLVGName := c.FindLVGForPVCBySelectedNode(pvc, pvc.Annotations[SelectedNodeAnnotation]) - // if targetLVGName != "" { - // c.UpdatePVC(targetLVGName, pvc) - // } - // } - // } - //} + pvcCh, found := lvgCh.(*lvgCache).pvcs[pvcKey] + if found { + c.log.Debug(fmt.Sprintf("[AddPVCToLVG] PVC %s cache has been already added to the LVMVolumeGroup %s. It will be updated", pvcKey, lvgName)) + pvcCh.pvc = pvc + } else { + c.log.Debug(fmt.Sprintf("[AddPVCToLVG] PVC %s cache was not found in LVMVolumeGroup %s. It will be added", pvcKey, lvgName)) + c.addNewPVC(lvgCh.(*lvgCache), pvc, lvgName) + } return nil } pvcKey := configurePVCKey(pvc) + + c.log.Debug(fmt.Sprintf("[AddPVCToLVG] add new PVC %s cache to the LVMVolumeGroup %s", pvcKey, lvgName)) lvgCh, found := c.lvgs.Load(lvgName) if !found { return fmt.Errorf("the LVMVolumeGroup %s was not found in the cache", lvgName) @@ -223,14 +193,6 @@ func (c *Cache) AddPVCToLVG(lvgName string, pvc *v1.PersistentVolumeClaim) error c.log.Debug(fmt.Sprintf("new cache will be initialized for PVC %s in LVMVolumeGroup %s", pvcKey, lvgName)) c.addNewPVC(lvgCh.(*lvgCache), pvc, lvgName) - //lvgCh.(*lvgCache).pvcs[pvcKey] = &pvcCache{pvc: pvc, selectedNode: ""} - // - //lvgsForPVC, _ := c.pvcLVGs.Load(pvcKey) - //if lvgsForPVC == nil { - // lvgsForPVC = make([]string, 0, 5) - //} - //lvgsForPVC = append(lvgsForPVC.([]string), lvgName) - //c.pvcLVGs.Store(pvcKey, lvgsForPVC) return nil } @@ -241,8 +203,9 @@ func (c *Cache) addNewPVC(lvgCh *lvgCache, pvc *v1.PersistentVolumeClaim, lvgNam lvgsForPVC, found := c.pvcLVGs.Load(pvcKey) if !found || lvgsForPVC == nil { - lvgsForPVC = make([]string, 0) + lvgsForPVC = make([]string, 0, lvgsPerPVCCount) } + c.log.Trace(fmt.Sprintf("[addNewPVC] LVMVolumeGroups from the cache for PVC %s before append: %v", pvcKey, lvgsForPVC)) lvgsForPVC = append(lvgsForPVC.([]string), lvgName) c.log.Trace(fmt.Sprintf("[addNewPVC] LVMVolumeGroups from the cache for PVC %s after append: %v", pvcKey, lvgsForPVC)) @@ -349,11 +312,6 @@ func (c *Cache) RemoveSpaceReservationForPVCWithSelectedNode(pvc *v1.PersistentV } for _, lvgName := range lvgArray.([]string) { - //// this check is needed in case some LVMVolumeGroup was removed from PVC lvgArray due to wipe PVC reservation space - //if lvgName == "" { - // continue - //} - lvgCh, found := c.lvgs.Load(lvgName) if !found || lvgCh == nil { err := fmt.Errorf("no cache found for the LVMVolumeGroup %s", lvgName) @@ -444,9 +402,9 @@ func (c *Cache) FindLVGForPVCBySelectedNode(pvc *v1.PersistentVolumeClaim, nodeN func (c *Cache) PrintTheCacheTraceLog() { c.log.Trace("*******************CACHE BEGIN*******************") - c.log.Trace("[LVMVolumeGroups]") + c.log.Trace("[LVMVolumeGroups BEGIN]") c.lvgs.Range(func(lvgName, lvgCh any) bool { - c.log.Trace(fmt.Sprintf("[LVMVolumeGroup: %s]", lvgName)) + c.log.Trace(fmt.Sprintf("[%s]", lvgName)) for pvcName, pvcCh := range lvgCh.(*lvgCache).pvcs { c.log.Trace(fmt.Sprintf(" PVC %s, selected node: %s", pvcName, pvcCh.selectedNode)) @@ -455,8 +413,8 @@ func (c *Cache) PrintTheCacheTraceLog() { return true }) - c.log.Trace("") - c.log.Trace("[PVC and LVG]") + c.log.Trace("[LVMVolumeGroups ENDS]") + c.log.Trace("[PVC and LVG BEGINS]") c.pvcLVGs.Range(func(pvcName, lvgs any) bool { c.log.Trace(fmt.Sprintf("[PVC: %s]", pvcName)) @@ -467,8 +425,8 @@ func (c *Cache) PrintTheCacheTraceLog() { return true }) - c.log.Trace("") - c.log.Trace("[Node and LVG]") + c.log.Trace("[PVC and LVG ENDS]") + c.log.Trace("[Node and LVG BEGINS]") c.nodeLVGs.Range(func(nodeName, lvgs any) bool { c.log.Trace(fmt.Sprintf("[Node: %s]", nodeName)) @@ -478,6 +436,7 @@ func (c *Cache) PrintTheCacheTraceLog() { return true }) + c.log.Trace("[Node and LVG ENDS]") c.log.Trace("*******************CACHE END*******************") } diff --git a/images/sds-local-volume-scheduler-extender/pkg/controller/lvg_watcher_cache.go b/images/sds-local-volume-scheduler-extender/pkg/controller/lvg_watcher_cache.go index ad42e2ab..ec028f75 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/controller/lvg_watcher_cache.go +++ b/images/sds-local-volume-scheduler-extender/pkg/controller/lvg_watcher_cache.go @@ -77,16 +77,16 @@ func RunLVGWatcherCacheController( log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to get all PVC for the LVMVolumeGroup %s", lvg.Name)) } for _, pvc := range pvcs { - log.Trace(fmt.Sprintf("[RunLVGWatcherCacheController] cached PVC %s belongs to LVMVolumeGroup %s", pvc.Name, lvg.Name)) + log.Trace(fmt.Sprintf("[RunLVGWatcherCacheController] cached PVC %s/%s belongs to LVMVolumeGroup %s", pvc.Namespace, pvc.Name, lvg.Name)) if pvc.Status.Phase == v1.ClaimBound { - log.Trace(fmt.Sprintf("[RunLVGWatcherCacheController] cached PVC %s has Status.Phase Bound. It will be removed from the cache for LVMVolumeGroup %s", pvc.Name, lvg.Name)) + log.Trace(fmt.Sprintf("[RunLVGWatcherCacheController] cached PVC %s/%s has Status.Phase Bound. It will be removed from the cache for LVMVolumeGroup %s", pvc.Namespace, pvc.Name, lvg.Name)) err = cache.RemoveBoundedPVCSpaceReservation(lvg.Name, pvc) if err != nil { - log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to remove PVC %s from the cache for the LVMVolumeGroup %s", pvc.Name, lvg.Name)) + log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to remove PVC %s/%s from the cache for the LVMVolumeGroup %s", pvc.Namespace, pvc.Name, lvg.Name)) continue } - log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] PVC %s was removed from the cache for LVMVolumeGroup %s", pvc.Name, lvg.Name)) + log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] PVC %s/%s was removed from the cache for LVMVolumeGroup %s", pvc.Namespace, pvc.Name, lvg.Name)) } } @@ -149,11 +149,11 @@ func RunLVGWatcherCacheController( log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] PVC %s/%s from the cache has Status.Phase Bound. It will be removed from the reserved space in the LVMVolumeGroup %s", pvc.Namespace, pvc.Name, newLvg.Name)) err = cache.RemoveBoundedPVCSpaceReservation(newLvg.Name, pvc) if err != nil { - log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to remove PVC %s from the cache in the LVMVolumeGroup %s", pvc.Name, newLvg.Name)) + log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to remove PVC %s/%s from the cache in the LVMVolumeGroup %s", pvc.Namespace, pvc.Name, newLvg.Name)) continue } - log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] PVC %s was removed from the LVMVolumeGroup %s in the cache", pvc.Name, newLvg.Name)) + log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] PVC %s/%s was removed from the LVMVolumeGroup %s in the cache", pvc.Namespace, pvc.Name, newLvg.Name)) } } From 834208ade1693427a2086fc9d15bd94c658c896e Mon Sep 17 00:00:00 2001 From: Viktor Kramarenko Date: Thu, 4 Apr 2024 23:01:31 +0300 Subject: [PATCH 15/21] minor fixes Signed-off-by: Viktor Kramarenko --- .../pkg/cache/cache.go | 15 ++++++++++++--- .../pkg/controller/pvc_watcher_cache.go | 2 -- .../pkg/scheduler/filter.go | 15 +++++++++------ 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go b/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go index 6995efe7..85cc6fa8 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go +++ b/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go @@ -132,7 +132,8 @@ func (c *Cache) AddPVCToLVG(lvgName string, pvc *v1.PersistentVolumeClaim) error return nil } - // this case will be triggered if the controller recovers after fail and finds some pending pvcs with selected nodes + // basically this case will be triggered if the controller recovers after fail and finds some pending pvcs with selected nodes, + // but also it might be true if kube scheduler will retry the request for the same PVC c.log.Trace(fmt.Sprintf("[AddPVCToLVG] PVC %s/%s annotations: %v", pvc.Namespace, pvc.Name, pvc.Annotations)) if pvc.Annotations[SelectedNodeAnnotation] != "" { c.log.Debug(fmt.Sprintf("PVC %s/%s has selected node anotation, selected node: %s", pvc.Namespace, pvc.Name, pvc.Annotations[SelectedNodeAnnotation])) @@ -180,14 +181,13 @@ func (c *Cache) AddPVCToLVG(lvgName string, pvc *v1.PersistentVolumeClaim) error if pvcCh, found := lvgCh.(*lvgCache).pvcs[pvcKey]; found { c.log.Debug(fmt.Sprintf("[AddPVCToLVG] PVC %s has been already added to the cache for the LVMVolumeGroup %s. It will be updated", pvcKey, lvgName)) - if pvcCh == nil { err := fmt.Errorf("cache is not initialized for PVC %s", pvcKey) c.log.Error(err, fmt.Sprintf("[AddPVCToLVG] an error occured while trying to add PVC %s to the cache", pvcKey)) return err } - pvcCh.pvc = pvc + c.UpdatePVC(lvgName, pvc) return nil } @@ -230,6 +230,15 @@ func (c *Cache) UpdatePVC(lvgName string, pvc *v1.PersistentVolumeClaim) error { lvgCh.(*lvgCache).pvcs[pvcKey].selectedNode = pvc.Annotations[SelectedNodeAnnotation] c.log.Debug(fmt.Sprintf("[UpdatePVC] successfully updated PVC %s with selected node %s in the cache for LVMVolumeGroup %s", pvcKey, pvc.Annotations[SelectedNodeAnnotation], lvgName)) + lvgsForPVC, found := c.pvcLVGs.Load(pvcKey) + if lvgsForPVC == nil || !found { + lvgsForPVC = make([]string, 0, lvgsPerPVCCount) + } + + if !slices2.Contains(lvgsForPVC.([]string), lvgName) { + lvgsForPVC = append(lvgsForPVC.([]string), lvgName) + } + return nil } diff --git a/images/sds-local-volume-scheduler-extender/pkg/controller/pvc_watcher_cache.go b/images/sds-local-volume-scheduler-extender/pkg/controller/pvc_watcher_cache.go index 5df84d68..a8dd0864 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/controller/pvc_watcher_cache.go +++ b/images/sds-local-volume-scheduler-extender/pkg/controller/pvc_watcher_cache.go @@ -204,8 +204,6 @@ func RunPVCWatcherCacheController( log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] successfully removed space reservation for PVC %s/%s with selected node", pvc.Namespace, pvc.Name)) schedulerCache.PrintTheCacheTraceLog() - //TODO: перенос удаления PVC в статусе Bound из кэша в данный контроллер вместо текущего в LVGWatcher - log.Info("[RunPVCWatcherCacheController] Update Func reconciliation ends") }, DeleteFunc: func(ctx context.Context, e event.DeleteEvent, q workqueue.RateLimitingInterface) { diff --git a/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go b/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go index 58bd6eae..da700ed3 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go +++ b/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go @@ -141,11 +141,11 @@ func populateCache(log logger.Logger, nodes []corev1.Node, pod *corev1.Pod, sche return err } - log.Trace(fmt.Sprintf("[populateCache] LVMVolumeGroups from Storage Class %s for PVC %s: %+v", sc.Name, volume.PersistentVolumeClaim.ClaimName, lvgsForPVC)) + log.Trace(fmt.Sprintf("[populateCache] LVMVolumeGroups from Storage Class %s for PVC %s/%s: %+v", sc.Name, pvc.Namespace, pvc.Name, lvgsForPVC)) for _, lvg := range lvgsForPVC { if slices.Contains(lvgNamesForTheNode, lvg.Name) { - log.Trace(fmt.Sprintf("[populateCache] PVC %s will reserve space in LVMVolumeGroup %s cache", volume.PersistentVolumeClaim.ClaimName, lvg.Name)) - err = schedulerCache.AddPVCToLVG(lvg.Name, pvcs[volume.PersistentVolumeClaim.ClaimName]) + log.Trace(fmt.Sprintf("[populateCache] PVC %s/%s will reserve space in LVMVolumeGroup %s cache", pvc.Namespace, pvc.Name, lvg.Name)) + err = schedulerCache.AddPVCToLVG(lvg.Name, pvc) if err != nil { return err } @@ -254,9 +254,10 @@ func filterNodes( usedLVGs := RemoveUnusedLVGs(lvgs, scLVGs) for _, lvg := range usedLVGs { - log.Trace(fmt.Sprintf("[filterNodes] the LVMVolumeGroup %s is actually used", lvg.Name)) + log.Trace(fmt.Sprintf("[filterNodes] the LVMVolumeGroup %s is actually used. VG size: %s, allocatedSize: %s", lvg.Name, lvg.Status.VGSize, lvg.Status.AllocatedSize)) } - lvgsThickFree, err := getLVGThickFreeSpaces(usedLVGs) + + lvgsThickFree, err := getLVGThickFreeSpaces(log, usedLVGs) if err != nil { return nil, err } @@ -404,14 +405,16 @@ func filterNodes( return result, nil } -func getLVGThickFreeSpaces(lvgs map[string]*v1alpha1.LvmVolumeGroup) (map[string]int64, error) { +func getLVGThickFreeSpaces(log logger.Logger, lvgs map[string]*v1alpha1.LvmVolumeGroup) (map[string]int64, error) { result := make(map[string]int64, len(lvgs)) for _, lvg := range lvgs { + log.Debug(fmt.Sprintf("[getLVGThickFreeSpaces] tries to count free VG space for LVMVolumeGroup %s", lvg.Name)) free, err := getVGFreeSpace(lvg) if err != nil { return nil, err } + log.Debug(fmt.Sprintf("[getLVGThickFreeSpaces] successfully counted free VG space for LVMVolumeGroup %s", lvg.Name)) result[lvg.Name] = free.Value() } From 43c227848eced11628d5f2257a308cca04bd7aa7 Mon Sep 17 00:00:00 2001 From: Viktor Kramarenko Date: Fri, 5 Apr 2024 13:12:37 +0300 Subject: [PATCH 16/21] minor fixes Signed-off-by: Viktor Kramarenko --- .../pkg/cache/cache.go | 20 +++++++++++--- .../pkg/scheduler/route.go | 26 +++++++++++++++++++ 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go b/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go index 85cc6fa8..d256d7fb 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go +++ b/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go @@ -159,10 +159,14 @@ func (c *Cache) AddPVCToLVG(lvgName string, pvc *v1.PersistentVolumeClaim) error return err } - pvcCh, found := lvgCh.(*lvgCache).pvcs[pvcKey] + _, found = lvgCh.(*lvgCache).pvcs[pvcKey] if found { c.log.Debug(fmt.Sprintf("[AddPVCToLVG] PVC %s cache has been already added to the LVMVolumeGroup %s. It will be updated", pvcKey, lvgName)) - pvcCh.pvc = pvc + err := c.UpdatePVC(lvgName, pvc) + if err != nil { + c.log.Error(err, fmt.Sprintf("[AddPVCToLVG] an error occured while trying to add PVC %s to the cache", pvcKey)) + return err + } } else { c.log.Debug(fmt.Sprintf("[AddPVCToLVG] PVC %s cache was not found in LVMVolumeGroup %s. It will be added", pvcKey, lvgName)) c.addNewPVC(lvgCh.(*lvgCache), pvc, lvgName) @@ -187,7 +191,11 @@ func (c *Cache) AddPVCToLVG(lvgName string, pvc *v1.PersistentVolumeClaim) error return err } - c.UpdatePVC(lvgName, pvc) + err := c.UpdatePVC(lvgName, pvc) + if err != nil { + c.log.Error(err, fmt.Sprintf("[AddPVCToLVG] an error occured while trying to add PVC %s to the cache", pvcKey)) + return err + } return nil } @@ -222,7 +230,11 @@ func (c *Cache) UpdatePVC(lvgName string, pvc *v1.PersistentVolumeClaim) error { if lvgCh.(*lvgCache).pvcs[pvcKey] == nil { c.log.Warning(fmt.Sprintf("[UpdatePVC] PVC %s was not found in the cache for the LVMVolumeGroup %s. It will be added", pvcKey, lvgName)) - c.AddPVCToLVG(lvgName, pvc) + err := c.AddPVCToLVG(lvgName, pvc) + if err != nil { + c.log.Error(err, fmt.Sprintf("[UpdatePVC] an error occurred while trying to update the PVC %s", pvcKey)) + return err + } return nil } diff --git a/images/sds-local-volume-scheduler-extender/pkg/scheduler/route.go b/images/sds-local-volume-scheduler-extender/pkg/scheduler/route.go index 1a02e090..3e86ab36 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/scheduler/route.go +++ b/images/sds-local-volume-scheduler-extender/pkg/scheduler/route.go @@ -31,6 +31,7 @@ type scheduler struct { client client.Client ctx context.Context cache *cache.Cache + requestCount int } func (s *scheduler) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -51,6 +52,10 @@ func (s *scheduler) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.log.Debug("[ServeHTTP] cache route starts handling the request") s.getCache(w, r) s.log.Debug("[ServeHTTP] cache route ends handling the request") + case "/stat": + s.log.Debug("[ServeHTTP] stat route starts handling the request") + s.getCacheStat(w, r) + s.log.Debug("[ServeHTTP] stat route ends handling the request") default: http.Error(w, "not found", http.StatusNotFound) } @@ -120,3 +125,24 @@ func (s *scheduler) getCache(w http.ResponseWriter, r *http.Request) { w.Write([]byte("unable to write the cache")) } } + +func (s *scheduler) getCacheStat(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + + pvcTotalCount := 0 + lvgs := s.cache.GetAllLVG() + for _, lvg := range lvgs { + pvcs, err := s.cache.GetAllPVCForLVG(lvg.Name) + if err != nil { + s.log.Error(err, "something bad") + } + + pvcTotalCount += len(pvcs) + } + + _, err := w.Write([]byte(fmt.Sprintf("Filter request count: %d , PVC Count from ALL LVG: %d", s.requestCount, pvcTotalCount))) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("unable to write the cache")) + } +} From 1597c0f5cfdfd212ce784e7f2696f213e78cf21c Mon Sep 17 00:00:00 2001 From: Viktor Kramarenko Date: Tue, 9 Apr 2024 14:10:15 +0300 Subject: [PATCH 17/21] minor fixes Signed-off-by: Viktor Kramarenko --- .../pkg/cache/cache.go | 76 +++--- .../pkg/cache/cache_test.go | 247 ++++++++++++++++++ .../pkg/scheduler/filter.go | 56 ++-- 3 files changed, 326 insertions(+), 53 deletions(-) create mode 100644 images/sds-local-volume-scheduler-extender/pkg/cache/cache_test.go diff --git a/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go b/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go index d256d7fb..244ab037 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go +++ b/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go @@ -24,7 +24,7 @@ type Cache struct { type lvgCache struct { lvg *v1alpha1.LvmVolumeGroup - pvcs map[string]*pvcCache + pvcs sync.Map } type pvcCache struct { @@ -41,7 +41,7 @@ func NewCache(logger logger.Logger) *Cache { func (c *Cache) AddLVG(lvg *v1alpha1.LvmVolumeGroup) { _, loaded := c.lvgs.LoadOrStore(lvg.Name, &lvgCache{ lvg: lvg, - pvcs: make(map[string]*pvcCache, pvcPerLVGCount), + pvcs: sync.Map{}, }) if loaded { c.log.Debug(fmt.Sprintf("[AddLVG] the LVMVolumeGroup %s has been already added to the cache", lvg.Name)) @@ -108,16 +108,12 @@ func (c *Cache) GetLVGReservedSpace(lvgName string) (int64, error) { c.log.Debug(fmt.Sprintf("[GetLVGReservedSpace] the LVMVolumeGroup %s was not found in the cache. Returns 0", lvgName)) return 0, nil } - if lvg.(*lvgCache).pvcs == nil { - err := fmt.Errorf("LVMVolumeGroup %s has no cached PVC", lvgName) - c.log.Error(err, fmt.Sprintf("[GetLVGReservedSpace] an error occurs for the LVMVolumeGroup %s", lvgName)) - return 0, err - } var space int64 - for _, pvc := range lvg.(*lvgCache).pvcs { - space += pvc.pvc.Spec.Resources.Requests.Storage().Value() - } + lvg.(*lvgCache).pvcs.Range(func(pvcName, pvcCh any) bool { + space = pvcCh.(*pvcCache).pvc.Spec.Resources.Requests.Storage().Value() + return true + }) return space, nil } @@ -159,7 +155,7 @@ func (c *Cache) AddPVCToLVG(lvgName string, pvc *v1.PersistentVolumeClaim) error return err } - _, found = lvgCh.(*lvgCache).pvcs[pvcKey] + _, found = lvgCh.(*lvgCache).pvcs.Load(pvcKey) if found { c.log.Debug(fmt.Sprintf("[AddPVCToLVG] PVC %s cache has been already added to the LVMVolumeGroup %s. It will be updated", pvcKey, lvgName)) err := c.UpdatePVC(lvgName, pvc) @@ -183,7 +179,7 @@ func (c *Cache) AddPVCToLVG(lvgName string, pvc *v1.PersistentVolumeClaim) error return fmt.Errorf("the LVMVolumeGroup %s was not found in the cache", lvgName) } - if pvcCh, found := lvgCh.(*lvgCache).pvcs[pvcKey]; found { + if pvcCh, found := lvgCh.(*lvgCache).pvcs.Load(pvcKey); found { c.log.Debug(fmt.Sprintf("[AddPVCToLVG] PVC %s has been already added to the cache for the LVMVolumeGroup %s. It will be updated", pvcKey, lvgName)) if pvcCh == nil { err := fmt.Errorf("cache is not initialized for PVC %s", pvcKey) @@ -207,7 +203,7 @@ func (c *Cache) AddPVCToLVG(lvgName string, pvc *v1.PersistentVolumeClaim) error func (c *Cache) addNewPVC(lvgCh *lvgCache, pvc *v1.PersistentVolumeClaim, lvgName string) { pvcKey := configurePVCKey(pvc) - lvgCh.pvcs[pvcKey] = &pvcCache{pvc: pvc, selectedNode: pvc.Annotations[SelectedNodeAnnotation]} + lvgCh.pvcs.Store(pvcKey, &pvcCache{pvc: pvc, selectedNode: pvc.Annotations[SelectedNodeAnnotation]}) lvgsForPVC, found := c.pvcLVGs.Load(pvcKey) if !found || lvgsForPVC == nil { @@ -228,7 +224,8 @@ func (c *Cache) UpdatePVC(lvgName string, pvc *v1.PersistentVolumeClaim) error { return fmt.Errorf("the LVMVolumeGroup %s was not found in the cache", lvgName) } - if lvgCh.(*lvgCache).pvcs[pvcKey] == nil { + _, found = lvgCh.(*lvgCache).pvcs.Load(pvcKey) + if !found { c.log.Warning(fmt.Sprintf("[UpdatePVC] PVC %s was not found in the cache for the LVMVolumeGroup %s. It will be added", pvcKey, lvgName)) err := c.AddPVCToLVG(lvgName, pvc) if err != nil { @@ -238,8 +235,12 @@ func (c *Cache) UpdatePVC(lvgName string, pvc *v1.PersistentVolumeClaim) error { return nil } - lvgCh.(*lvgCache).pvcs[pvcKey].pvc = pvc - lvgCh.(*lvgCache).pvcs[pvcKey].selectedNode = pvc.Annotations[SelectedNodeAnnotation] + newPVCCh := &pvcCache{ + pvc: pvc, + selectedNode: pvc.Annotations[SelectedNodeAnnotation], + } + + lvgCh.(*lvgCache).pvcs.Store(pvcKey, newPVCCh) c.log.Debug(fmt.Sprintf("[UpdatePVC] successfully updated PVC %s with selected node %s in the cache for LVMVolumeGroup %s", pvcKey, pvc.Annotations[SelectedNodeAnnotation], lvgName)) lvgsForPVC, found := c.pvcLVGs.Load(pvcKey) @@ -262,10 +263,11 @@ func (c *Cache) GetAllPVCForLVG(lvgName string) ([]*v1.PersistentVolumeClaim, er return nil, err } - result := make([]*v1.PersistentVolumeClaim, 0, len(lvgCh.(*lvgCache).pvcs)) - for _, pvcCh := range lvgCh.(*lvgCache).pvcs { - result = append(result, pvcCh.pvc) - } + result := make([]*v1.PersistentVolumeClaim, 0, pvcPerLVGCount) + lvgCh.(*lvgCache).pvcs.Range(func(pvcName, pvcCh any) bool { + result = append(result, pvcCh.(*pvcCache).pvc) + return true + }) return result, nil } @@ -279,13 +281,14 @@ func (c *Cache) GetPVCSelectedNodeName(lvgName string, pvc *v1.PersistentVolumeC return "", err } - if lvgCh.(*lvgCache).pvcs[pvcKey] == nil { + pvcCh, found := lvgCh.(*lvgCache).pvcs.Load(pvcKey) + if !found { err := fmt.Errorf("cache was not found for PVC %s", pvcKey) c.log.Error(err, fmt.Sprintf("[GetPVCSelectedNodeName] an error occured while trying to get selected node name for the PVC %s in the LVMVolumeGroup %s", pvcKey, lvgName)) return "", err } - return lvgCh.(*lvgCache).pvcs[pvcKey].selectedNode, nil + return pvcCh.(*pvcCache).selectedNode, nil } func (c *Cache) GetLVGNamesForPVC(pvc *v1.PersistentVolumeClaim) []string { @@ -309,19 +312,28 @@ func (c *Cache) RemoveBoundedPVCSpaceReservation(lvgName string, pvc *v1.Persist return err } - pvcCh, found := lvgCh.(*lvgCache).pvcs[pvcKey] + pvcCh, found := lvgCh.(*lvgCache).pvcs.Load(pvcKey) if !found || pvcCh == nil { err := fmt.Errorf("cache for PVC %s was not found", pvcKey) c.log.Error(err, fmt.Sprintf("[RemoveBoundedPVCSpaceReservation] an error occured while trying to remove space reservation for PVC %s in the LVMVolumeGroup %s", pvcKey, lvgName)) return err } - delete(lvgCh.(*lvgCache).pvcs, pvcKey) + lvgCh.(*lvgCache).pvcs.Delete(pvcKey) c.pvcLVGs.Delete(pvcKey) return nil } +func (c *Cache) CheckIsPVCCached(pvc *v1.PersistentVolumeClaim) bool { + pvcKey := configurePVCKey(pvc) + if _, found := c.pvcLVGs.Load(pvcKey); found { + return true + } + + return false +} + func (c *Cache) RemoveSpaceReservationForPVCWithSelectedNode(pvc *v1.PersistentVolumeClaim) error { pvcKey := configurePVCKey(pvc) selectedLVGName := "" @@ -340,14 +352,15 @@ func (c *Cache) RemoveSpaceReservationForPVCWithSelectedNode(pvc *v1.PersistentV return err } - if _, found := lvgCh.(*lvgCache).pvcs[pvcKey]; !found { + pvcCh, found := lvgCh.(*lvgCache).pvcs.Load(pvcKey) + if !found { c.log.Debug(fmt.Sprintf("[RemoveSpaceReservationForPVCWithSelectedNode] PVC %s space reservation in the LVMVolumeGroup %s has been already removed", pvcKey, lvgName)) continue } - selectedNode := lvgCh.(*lvgCache).pvcs[pvcKey].selectedNode + selectedNode := pvcCh.(*pvcCache).selectedNode if selectedNode == "" { - delete(lvgCh.(*lvgCache).pvcs, pvcKey) + lvgCh.(*lvgCache).pvcs.Delete(pvcKey) c.log.Debug(fmt.Sprintf("[RemoveSpaceReservationForPVCWithSelectedNode] removed space reservation for PVC %s in the LVMVolumeGroup %s due the PVC got selected to the node %s", pvcKey, lvgName, selectedNode)) } else { selectedLVGName = lvgName @@ -381,7 +394,7 @@ func (c *Cache) RemovePVCSpaceReservationForced(pvc *v1.PersistentVolumeClaim) { for _, lvgName := range lvgArray.([]string) { lvgCh, found := c.lvgs.Load(lvgName) if found { - delete(lvgCh.(*lvgCache).pvcs, pvcKey.(string)) + lvgCh.(*lvgCache).pvcs.Delete(pvcKey.(string)) } } } @@ -427,9 +440,10 @@ func (c *Cache) PrintTheCacheTraceLog() { c.lvgs.Range(func(lvgName, lvgCh any) bool { c.log.Trace(fmt.Sprintf("[%s]", lvgName)) - for pvcName, pvcCh := range lvgCh.(*lvgCache).pvcs { - c.log.Trace(fmt.Sprintf(" PVC %s, selected node: %s", pvcName, pvcCh.selectedNode)) - } + lvgCh.(*lvgCache).pvcs.Range(func(pvcName, pvcCh any) bool { + c.log.Trace(fmt.Sprintf(" PVC %s, selected node: %s", pvcName, pvcCh.(*pvcCache).selectedNode)) + return true + }) return true }) diff --git a/images/sds-local-volume-scheduler-extender/pkg/cache/cache_test.go b/images/sds-local-volume-scheduler-extender/pkg/cache/cache_test.go new file mode 100644 index 00000000..3f86869d --- /dev/null +++ b/images/sds-local-volume-scheduler-extender/pkg/cache/cache_test.go @@ -0,0 +1,247 @@ +package cache + +import ( + "fmt" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sds-local-volume-scheduler-extender/api/v1alpha1" + "sds-local-volume-scheduler-extender/pkg/logger" + "testing" +) + +func BenchmarkCache_DeleteLVG(b *testing.B) { + cache := NewCache(logger.Logger{}) + lvg := &v1alpha1.LvmVolumeGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "first", + }, + } + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + cache.AddLVG(lvg) + if _, found := cache.lvgs.Load(lvg.Name); found { + //b.Log("lvg found, delete it") + cache.DeleteLVG(lvg.Name) + } + } + }) +} + +func BenchmarkCache_GetLVGReservedSpace(b *testing.B) { + cache := NewCache(logger.Logger{}) + lvg := &v1alpha1.LvmVolumeGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "first", + }, + } + + cache.AddLVG(lvg) + + pvcs := []v1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pvc-1", + }, + Spec: v1.PersistentVolumeClaimSpec{ + Resources: v1.VolumeResourceRequirements{ + Requests: v1.ResourceList{ + "pvc": *resource.NewQuantity(1000000, resource.BinarySI), + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pvc-2", + }, + Spec: v1.PersistentVolumeClaimSpec{ + Resources: v1.VolumeResourceRequirements{ + Requests: v1.ResourceList{ + "pvc": *resource.NewQuantity(2000000, resource.BinarySI), + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pvc-3", + }, + Spec: v1.PersistentVolumeClaimSpec{ + Resources: v1.VolumeResourceRequirements{ + Requests: v1.ResourceList{ + "pvc": *resource.NewQuantity(30000000, resource.BinarySI), + }, + }, + }, + }, + } + + for _, pvc := range pvcs { + cache.AddPVCToLVG(lvg.Name, &pvc) + } + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _, err := cache.GetLVGReservedSpace(lvg.Name) + if err != nil { + b.Error(err) + } + } + }) +} + +func BenchmarkCache_GetAllLVG(b *testing.B) { + cache := NewCache(logger.Logger{}) + lvgs := map[string]*lvgCache{ + "first": { + lvg: &v1alpha1.LvmVolumeGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "first", + }, + }, + }, + "second": { + lvg: &v1alpha1.LvmVolumeGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "second", + }, + }, + }, + } + + for _, lvg := range lvgs { + cache.lvgs.Store(lvg.lvg.Name, lvg) + } + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + mp := cache.GetAllLVG() + + if len(mp) != 2 { + b.Error("not enough LVG") + } + } + }) +} + +func BenchmarkCache_GetLVGNamesByNodeName(b *testing.B) { + cache := NewCache(logger.Logger{}) + lvgs := []string{ + "first", + "second", + "third", + } + nodeName := "test-node" + + cache.nodeLVGs.Store(nodeName, lvgs) + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + l := cache.GetLVGNamesByNodeName(nodeName) + if len(l) != 3 { + b.Error("not enough LVG") + } + } + }) +} + +func BenchmarkCache_TryGetLVG(b *testing.B) { + cache := NewCache(logger.Logger{}) + name := "test-name" + + lvg := &v1alpha1.LvmVolumeGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + cache.AddLVG(lvg) + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + l := cache.TryGetLVG(lvg.Name) + if l == nil { + b.Error("nil LVG from cache") + } + } + }) +} + +func BenchmarkCache_AddLVG(b *testing.B) { + cache := NewCache(logger.Logger{}) + i := 0 + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + i++ + lvg := &v1alpha1.LvmVolumeGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("test-lvg-%d", i), + }, + } + cache.AddLVG(lvg) + } + }) +} + +func BenchmarkCache_UpdateLVG(b *testing.B) { + cache := NewCache(logger.Logger{}) + name := "test-name" + i := 0 + + lvg := &v1alpha1.LvmVolumeGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + cache.AddLVG(lvg) + + _, found := cache.lvgs.Load(name) + if !found { + b.Error("not found LVG") + } + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + i++ + updated := &v1alpha1.LvmVolumeGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Status: v1alpha1.LvmVolumeGroupStatus{ + AllocatedSize: fmt.Sprintf("2%dGi", i), + }, + } + b.Logf("updates the LVG with allocated size: %s", updated.Status.AllocatedSize) + cache.UpdateLVG(updated) + } + }) +} + +func BenchmarkCache_UpdatePVC(b *testing.B) { + cache := NewCache(logger.Logger{}) + i := 0 + lvg := &v1alpha1.LvmVolumeGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-lvg", + }, + } + cache.AddLVG(lvg) + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + i++ + pvc := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("test-pvc-%d", i), + Namespace: "test-ns", + }, + } + err := cache.UpdatePVC(lvg.Name, pvc) + if err != nil { + b.Error(err) + } + } + }) +} diff --git a/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go b/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go index da700ed3..285fa55a 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go +++ b/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go @@ -54,8 +54,10 @@ func (s *scheduler) filter(w http.ResponseWriter, r *http.Request) { return } + s.log.Debug(fmt.Sprintf("[filter] starts the filtering for Pod %s/%s", input.Pod.Namespace, input.Pod.Name)) + for _, n := range input.Nodes.Items { - s.log.Trace(fmt.Sprintf("[filter] a node from request, name :%s", n.Name)) + s.log.Trace(fmt.Sprintf("[filter] Pod %s/%s has node %s from the request", input.Pod.Namespace, input.Pod.Name, n.Name)) } pvcs, err := getUsedPVC(s.ctx, s.client, s.log, input.Pod) @@ -71,7 +73,14 @@ func (s *scheduler) filter(w http.ResponseWriter, r *http.Request) { } for _, pvc := range pvcs { - s.log.Trace(fmt.Sprintf("[filter] Pod %s/%s used PVC: %s", input.Pod.Namespace, input.Pod.Name, pvc.Name)) + s.log.Trace(fmt.Sprintf("[filter] Pod %s/%s uses PVC: %s", input.Pod.Namespace, input.Pod.Name, pvc.Name)) + + if s.cache.CheckIsPVCCached(pvc) { + s.log.Debug(fmt.Sprintf("[filter] PVC %s/%s has been already stored in the cache. It will be removed due the conflicts", pvc.Namespace, pvc.Name)) + s.cache.RemovePVCSpaceReservationForced(pvc) + } else { + s.log.Debug(fmt.Sprintf("[filter] PVC %s/%s was not found in the cache", pvc.Namespace, pvc.Name)) + } } scs, err := getStorageClassesUsedByPVCs(s.ctx, s.client, pvcs) @@ -81,28 +90,29 @@ func (s *scheduler) filter(w http.ResponseWriter, r *http.Request) { return } for _, sc := range scs { - s.log.Trace(fmt.Sprintf("[filter] used StorageClasses: %s", sc.Name)) + s.log.Trace(fmt.Sprintf("[filter] Pod %s/%s uses StorageClass: %s", input.Pod.Namespace, input.Pod.Name, sc.Name)) } - s.log.Debug("[filter] starts to extract pvcRequests size") + s.log.Debug(fmt.Sprintf("[filter] starts to extract PVC requested sizes for a Pod %s/%s", input.Pod.Namespace, input.Pod.Name)) pvcRequests, err := extractRequestedSize(s.ctx, s.client, s.log, pvcs, scs) if err != nil { - s.log.Error(err, fmt.Sprintf("[filter] unable to extract request size for a pod %s", input.Pod.Name)) + s.log.Error(err, fmt.Sprintf("[filter] unable to extract request size for a Pod %s/%s", input.Pod.Namespace, input.Pod.Name)) http.Error(w, "bad request", http.StatusBadRequest) return } - s.log.Debug("[filter] successfully extracted the pvcRequests size") + s.log.Debug(fmt.Sprintf("[filter] successfully extracted the PVC requested sizes of a Pod %s/%s", input.Pod.Namespace, input.Pod.Name)) - s.log.Debug("[filter] starts to filter the nodes") - filteredNodes, err := filterNodes(s.log, s.cache, input.Nodes, pvcs, scs, pvcRequests) + s.log.Debug(fmt.Sprintf("[filter] starts to filter the nodes from the request for a Pod %s/%s", input.Pod.Namespace, input.Pod.Name)) + filteredNodes, err := filterNodes(s.log, s.cache, input.Nodes, input.Pod, pvcs, scs, pvcRequests) if err != nil { s.log.Error(err, "[filter] unable to filter the nodes") http.Error(w, "bad request", http.StatusBadRequest) return } - s.log.Debug("[filter] successfully filtered the nodes") + s.log.Debug(fmt.Sprintf("[filter] successfully filtered the nodes from the request for a Pod %s/%s", input.Pod.Namespace, input.Pod.Name)) - s.log.Debug("[filter] starts to populate the cache") + s.log.Debug(fmt.Sprintf("[filter] starts to populate the cache for a Pod %s/%s", input.Pod.Namespace, input.Pod.Name)) + s.log.Trace(fmt.Sprintf("[filter] cache before the PVC reservation for a Pod %s/%s", input.Pod.Namespace, input.Pod.Name)) s.cache.PrintTheCacheTraceLog() err = populateCache(s.log, filteredNodes.Nodes.Items, input.Pod, s.cache, pvcs, scs) if err != nil { @@ -110,7 +120,8 @@ func (s *scheduler) filter(w http.ResponseWriter, r *http.Request) { http.Error(w, "bad request", http.StatusBadRequest) return } - s.log.Debug("[filter] successfully populated the cache") + s.log.Debug(fmt.Sprintf("[filter] successfully populated the cache for a Pod %s/%s", input.Pod.Namespace, input.Pod.Name)) + s.log.Trace(fmt.Sprintf("[filter] cache after the PVC reservation for a Pod %s/%s", input.Pod.Namespace, input.Pod.Name)) s.cache.PrintTheCacheTraceLog() w.Header().Set("content-type", "application/json") @@ -120,7 +131,7 @@ func (s *scheduler) filter(w http.ResponseWriter, r *http.Request) { http.Error(w, "internal error", http.StatusInternalServerError) return } - s.log.Debug("[filter] ends the serving") + s.log.Debug(fmt.Sprintf("[filter] ends the serving the request for a Pod %s/%s", input.Pod.Namespace, input.Pod.Name)) } func populateCache(log logger.Logger, nodes []corev1.Node, pod *corev1.Pod, schedulerCache *cache.Cache, pvcs map[string]*corev1.PersistentVolumeClaim, scs map[string]*v1.StorageClass) error { @@ -224,6 +235,7 @@ func filterNodes( log logger.Logger, schedulerCache *cache.Cache, nodes *corev1.NodeList, + pod *corev1.Pod, pvcs map[string]*corev1.PersistentVolumeClaim, scs map[string]*v1.StorageClass, pvcRequests map[string]PVCRequest, @@ -240,12 +252,12 @@ func filterNodes( log.Trace(fmt.Sprintf("[filterNodes] LVMVolumeGroup %s in the cache", lvg.Name)) } - log.Debug("[filterNodes] starts to get LVMVolumeGroups for Storage Classes") + log.Debug(fmt.Sprintf("[filterNodes] starts to get LVMVolumeGroups for Storage Classes for a Pod %s/%s", pod.Namespace, pod.Name)) scLVGs, err := GetSortedLVGsFromStorageClasses(scs) if err != nil { return nil, err } - log.Debug("[filterNodes] successfully got LVMVolumeGroups for Storage Classes") + log.Debug(fmt.Sprintf("[filterNodes] successfully got LVMVolumeGroups for Storage Classes for a Pod %s/%s", pod.Namespace, pod.Name)) for scName, sortedLVGs := range scLVGs { for _, lvg := range sortedLVGs { log.Trace(fmt.Sprintf("[filterNodes] LVMVolumeGroup %s belongs to Storage Class %s", lvg.Name, scName)) @@ -261,7 +273,7 @@ func filterNodes( if err != nil { return nil, err } - log.Trace(fmt.Sprintf("[filterNodes] current LVMVolumeGroups Thick FreeSpace on the node: %+v", lvgsThickFree)) + log.Trace(fmt.Sprintf("[filterNodes] for a Pod %s/%s current LVMVolumeGroups Thick FreeSpace on the node: %+v", pod.Namespace, pod.Name, lvgsThickFree)) for lvgName, freeSpace := range lvgsThickFree { log.Trace(fmt.Sprintf("[filterNodes] current LVMVolumeGroup %s Thick free space %s", lvgName, resource.NewQuantity(freeSpace, resource.BinarySI))) @@ -273,7 +285,7 @@ func filterNodes( log.Trace(fmt.Sprintf("[filterNodes] current LVMVolumeGroup %s reserved PVC space %s", lvgName, resource.NewQuantity(reservedSize, resource.BinarySI))) lvgsThickFree[lvgName] -= reservedSize } - log.Trace(fmt.Sprintf("[filterNodes] current LVMVolumeGroups Thick FreeSpace with reserved PVC: %+v", lvgsThickFree)) + log.Trace(fmt.Sprintf("[filterNodes] for a Pod %s/%s current LVMVolumeGroups Thick FreeSpace with reserved PVC: %+v", pod.Namespace, pod.Name, lvgsThickFree)) lvgsThickFreeMutex := &sync.RWMutex{} @@ -395,11 +407,11 @@ func filterNodes( } for _, node := range result.Nodes.Items { - log.Trace(fmt.Sprintf("[filterNodes] suitable node: %s", node.Name)) + log.Trace(fmt.Sprintf("[filterNodes] for a Pod %s/%s there is a suitable node: %s", pod.Namespace, pod.Name, node.Name)) } for node, reason := range result.FailedNodes { - log.Trace(fmt.Sprintf("[filterNodes] failed node: %s, reason: %s", node, reason)) + log.Trace(fmt.Sprintf("[filterNodes] for a Pod %s/%s there is a failed node: %s, reason: %s", pod.Namespace, pod.Name, node, reason)) } return result, nil @@ -635,7 +647,7 @@ func getUsedPVC(ctx context.Context, cl client.Client, log logger.Logger, pod *c for _, volume := range pod.Spec.Volumes { if volume.PersistentVolumeClaim != nil { - log.Trace(fmt.Sprintf("[getUsedPVC] Pod %s uses PVC %s", pod.Name, volume.PersistentVolumeClaim.ClaimName)) + log.Trace(fmt.Sprintf("[getUsedPVC] Pod %s/%s uses PVC %s", pod.Namespace, pod.Name, volume.PersistentVolumeClaim.ClaimName)) usedPvc[volume.PersistentVolumeClaim.ClaimName] = pvcMap[volume.PersistentVolumeClaim.ClaimName] } } @@ -647,7 +659,7 @@ func getUsedPVC(ctx context.Context, cl client.Client, log logger.Logger, pod *c if volume.PersistentVolumeClaim != nil { if _, added := usedPvc[volume.PersistentVolumeClaim.ClaimName]; !added { filled = false - log.Warning(fmt.Sprintf("[getUsedPVC] PVC %s was not found in the cache for Pod %s", volume.PersistentVolumeClaim.ClaimName, pod.Name)) + log.Warning(fmt.Sprintf("[getUsedPVC] PVC %s was not found in the cache for Pod %s/%s", volume.PersistentVolumeClaim.ClaimName, pod.Namespace, pod.Name)) break } } @@ -655,13 +667,13 @@ func getUsedPVC(ctx context.Context, cl client.Client, log logger.Logger, pod *c } if !filled { - log.Warning(fmt.Sprintf("[getUsedPVC] some PVCs were not found in the cache for Pod %s. Retry to find them again.", pod.Name)) + log.Warning(fmt.Sprintf("[getUsedPVC] some PVCs were not found in the cache for Pod %s/%s. Retry to find them again.", pod.Namespace, pod.Name)) time.Sleep(100 * time.Millisecond) continue } if filled { - log.Debug(fmt.Sprintf("[getUsedPVC] Every PVC for Pod %s was found in the cache", pod.Name)) + log.Debug(fmt.Sprintf("[getUsedPVC] Every PVC for Pod %s/%s was found in the cache", pod.Namespace, pod.Name)) break } } From dfefec516fdd725530bcc92203551c0e89a26eeb Mon Sep 17 00:00:00 2001 From: Viktor Kramarenko Date: Tue, 9 Apr 2024 17:00:40 +0300 Subject: [PATCH 18/21] minor fixes Signed-off-by: Viktor Kramarenko --- .../pkg/scheduler/filter.go | 55 +++++++++---------- .../pkg/scheduler/prioritize.go | 20 ++++--- .../pkg/scheduler/route.go | 31 +++++++++++ .../rbac-for-us.yaml | 4 +- 4 files changed, 71 insertions(+), 39 deletions(-) diff --git a/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go b/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go index 285fa55a..ef59cef4 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go +++ b/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go @@ -32,7 +32,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/yaml" "sync" - "time" ) const ( @@ -79,7 +78,7 @@ func (s *scheduler) filter(w http.ResponseWriter, r *http.Request) { s.log.Debug(fmt.Sprintf("[filter] PVC %s/%s has been already stored in the cache. It will be removed due the conflicts", pvc.Namespace, pvc.Name)) s.cache.RemovePVCSpaceReservationForced(pvc) } else { - s.log.Debug(fmt.Sprintf("[filter] PVC %s/%s was not found in the cache", pvc.Namespace, pvc.Name)) + s.log.Debug(fmt.Sprintf("[filter] PVC %s/%s was not found in the scheduler cache", pvc.Namespace, pvc.Name)) } } @@ -136,10 +135,9 @@ func (s *scheduler) filter(w http.ResponseWriter, r *http.Request) { func populateCache(log logger.Logger, nodes []corev1.Node, pod *corev1.Pod, schedulerCache *cache.Cache, pvcs map[string]*corev1.PersistentVolumeClaim, scs map[string]*v1.StorageClass) error { for _, node := range nodes { - log.Debug(fmt.Sprintf("[populateCache] starts the work for node %s", node.Name)) for _, volume := range pod.Spec.Volumes { if volume.PersistentVolumeClaim != nil { - log.Debug(fmt.Sprintf("[populateCache] reconcile the PVC %s for Pod %s/%s", volume.PersistentVolumeClaim.ClaimName, pod.Namespace, pod.Name)) + log.Debug(fmt.Sprintf("[populateCache] reconcile the PVC %s for Pod %s/%s on node %s", volume.PersistentVolumeClaim.ClaimName, pod.Namespace, pod.Name, node.Name)) lvgNamesForTheNode := schedulerCache.GetLVGNamesByNodeName(node.Name) log.Trace(fmt.Sprintf("[populateCache] LVMVolumeGroups from cache for the node %s: %v", node.Name, lvgNamesForTheNode)) pvc := pvcs[volume.PersistentVolumeClaim.ClaimName] @@ -192,6 +190,7 @@ func extractRequestedSize( pvcRequests := make(map[string]PVCRequest, len(pvcs)) for _, pvc := range pvcs { sc := scs[*pvc.Spec.StorageClassName] + log.Debug(fmt.Sprintf("[extractRequestedSize] PVC %s/%s has status phase: %s", pvc.Namespace, pvc.Name, pvc.Status.Phase)) switch pvc.Status.Phase { case corev1.ClaimPending: switch sc.Parameters[lvmTypeParamKey] { @@ -652,30 +651,30 @@ func getUsedPVC(ctx context.Context, cl client.Client, log logger.Logger, pod *c } } - filled := false - if len(pvcMap) > 0 { - filled = true - for _, volume := range pod.Spec.Volumes { - if volume.PersistentVolumeClaim != nil { - if _, added := usedPvc[volume.PersistentVolumeClaim.ClaimName]; !added { - filled = false - log.Warning(fmt.Sprintf("[getUsedPVC] PVC %s was not found in the cache for Pod %s/%s", volume.PersistentVolumeClaim.ClaimName, pod.Namespace, pod.Name)) - break - } - } - } - } - - if !filled { - log.Warning(fmt.Sprintf("[getUsedPVC] some PVCs were not found in the cache for Pod %s/%s. Retry to find them again.", pod.Namespace, pod.Name)) - time.Sleep(100 * time.Millisecond) - continue - } - - if filled { - log.Debug(fmt.Sprintf("[getUsedPVC] Every PVC for Pod %s/%s was found in the cache", pod.Namespace, pod.Name)) - break - } + //filled := false + //if len(pvcMap) > 0 { + // filled = true + // for _, volume := range pod.Spec.Volumes { + // if volume.PersistentVolumeClaim != nil { + // if _, added := usedPvc[volume.PersistentVolumeClaim.ClaimName]; !added { + // filled = false + // log.Warning(fmt.Sprintf("[getUsedPVC] PVC %s was not found in the cache for Pod %s/%s", volume.PersistentVolumeClaim.ClaimName, pod.Namespace, pod.Name)) + // break + // } + // } + // } + //} + // + //if !filled { + // log.Warning(fmt.Sprintf("[getUsedPVC] some PVCs were not found in the cache for Pod %s/%s. Retry to find them again.", pod.Namespace, pod.Name)) + // time.Sleep(100 * time.Millisecond) + // continue + //} + // + //if filled { + // log.Debug(fmt.Sprintf("[getUsedPVC] Every PVC for Pod %s/%s was found in the cache", pod.Namespace, pod.Name)) + // break + //} } return usedPvc, err diff --git a/images/sds-local-volume-scheduler-extender/pkg/scheduler/prioritize.go b/images/sds-local-volume-scheduler-extender/pkg/scheduler/prioritize.go index e6459646..9e09b0bf 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/scheduler/prioritize.go +++ b/images/sds-local-volume-scheduler-extender/pkg/scheduler/prioritize.go @@ -42,6 +42,8 @@ func (s *scheduler) prioritize(w http.ResponseWriter, r *http.Request) { return } + s.log.Debug(fmt.Sprintf("[prioritize] starts the prioritizing for Pod %s/%s", input.Pod.Namespace, input.Pod.Name)) + pvcs, err := getUsedPVC(s.ctx, s.client, s.log, input.Pod) if err != nil { s.log.Error(err, "[prioritize] unable to get PVC from the Pod") @@ -54,7 +56,7 @@ func (s *scheduler) prioritize(w http.ResponseWriter, r *http.Request) { return } for _, pvc := range pvcs { - s.log.Trace(fmt.Sprintf("[prioritize] used PVC: %s", pvc.Name)) + s.log.Trace(fmt.Sprintf("[prioritize] Pod %s/%s uses PVC: %s", input.Pod.Namespace, input.Pod.Name, pvc.Name)) } scs, err := getStorageClassesUsedByPVCs(s.ctx, s.client, pvcs) @@ -64,30 +66,30 @@ func (s *scheduler) prioritize(w http.ResponseWriter, r *http.Request) { return } for _, sc := range scs { - s.log.Trace(fmt.Sprintf("[prioritize] used StorageClasses: %s", sc.Name)) + s.log.Trace(fmt.Sprintf("[prioritize] Pod %s/%s uses Storage Class: %s", input.Pod.Namespace, input.Pod.Name, sc.Name)) } - s.log.Debug("[prioritize] starts to extract pvcRequests size") + s.log.Debug(fmt.Sprintf("[prioritize] starts to extract pvcRequests size for Pod %s/%s", input.Pod.Namespace, input.Pod.Name)) pvcRequests, err := extractRequestedSize(s.ctx, s.client, s.log, pvcs, scs) if err != nil { s.log.Error(err, fmt.Sprintf("[filter] unable to extract request size for a pod %s", input.Pod.Name)) http.Error(w, "bad request", http.StatusBadRequest) } - s.log.Debug("[filter] successfully extracted the pvcRequests size") + s.log.Debug(fmt.Sprintf("[filter] successfully extracted the pvcRequests size for Pod %s/%s", input.Pod.Namespace, input.Pod.Name)) - s.log.Debug("[prioritize] starts to score the nodes") + s.log.Debug(fmt.Sprintf("[prioritize] starts to score the nodes for Pod %s/%s", input.Pod.Namespace, input.Pod.Name)) result, err := scoreNodes(s.log, s.cache, input.Nodes, pvcs, scs, pvcRequests, s.defaultDivisor) if err != nil { - s.log.Error(err, "[prioritize] unable to score nodes") - http.Error(w, "Bad Request.", http.StatusBadRequest) + s.log.Error(err, fmt.Sprintf("[prioritize] unable to score nodes for Pod %s/%s", input.Pod.Namespace, input.Pod.Name)) + http.Error(w, "bad request", http.StatusBadRequest) return } - s.log.Debug("[prioritize] successfully scored the nodes") + s.log.Debug(fmt.Sprintf("[prioritize] successfully scored the nodes for Pod %s/%s", input.Pod.Namespace, input.Pod.Name)) w.Header().Set("content-type", "application/json") err = json.NewEncoder(w).Encode(result) if err != nil { - s.log.Error(err, "[prioritize] unable to encode a response") + s.log.Error(err, fmt.Sprintf("[prioritize] unable to encode a response for a Pod %s/%s", input.Pod.Namespace, input.Pod.Name)) http.Error(w, "internal error", http.StatusInternalServerError) } s.log.Debug("[prioritize] ends serving") diff --git a/images/sds-local-volume-scheduler-extender/pkg/scheduler/route.go b/images/sds-local-volume-scheduler-extender/pkg/scheduler/route.go index 3e86ab36..b40df467 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/scheduler/route.go +++ b/images/sds-local-volume-scheduler-extender/pkg/scheduler/route.go @@ -19,6 +19,7 @@ package scheduler import ( "context" "fmt" + "k8s.io/apimachinery/pkg/api/resource" "net/http" "sds-local-volume-scheduler-extender/pkg/cache" "sds-local-volume-scheduler-extender/pkg/logger" @@ -56,6 +57,10 @@ func (s *scheduler) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.log.Debug("[ServeHTTP] stat route starts handling the request") s.getCacheStat(w, r) s.log.Debug("[ServeHTTP] stat route ends handling the request") + case "/reserved": + s.log.Debug("[ServeHTTP] reserved route starts handling the request") + s.getReservedSpace(w, r) + s.log.Debug("[ServeHTTP] reserved route ends handling the request") default: http.Error(w, "not found", http.StatusNotFound) } @@ -146,3 +151,29 @@ func (s *scheduler) getCacheStat(w http.ResponseWriter, r *http.Request) { w.Write([]byte("unable to write the cache")) } } + +func (s *scheduler) getReservedSpace(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + + lvgs := s.cache.GetAllLVG() + + result := make(map[string]int64, len(lvgs)) + for _, lvg := range lvgs { + space, err := s.cache.GetLVGReservedSpace(lvg.Name) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("unable to get the space")) + } + + result[lvg.Name] = space + } + + for lvgName, space := range result { + _, err := w.Write([]byte(fmt.Sprintf("LVMVolumeGroup: %s, Reserved space: %s\n", lvgName, resource.NewQuantity(space, resource.BinarySI)))) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("unable to write the cache")) + } + } + +} diff --git a/templates/sds-local-volume-scheduler-extender/rbac-for-us.yaml b/templates/sds-local-volume-scheduler-extender/rbac-for-us.yaml index b2b7fa07..6687e6d9 100644 --- a/templates/sds-local-volume-scheduler-extender/rbac-for-us.yaml +++ b/templates/sds-local-volume-scheduler-extender/rbac-for-us.yaml @@ -46,10 +46,10 @@ rules: verbs: ["create", "get", "update"] - apiGroups: [ "storage.deckhouse.io" ] resources: [ "lvmvolumegroups" ] - verbs: [ "list", "watch"] + verbs: [ "list", "watch", "get"] - apiGroups: ["v1"] resources: ["persistentvolumeclaims"] - verbs: ["list", "watch"] + verbs: ["list", "watch", "get"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding From dae353b0ea790181a32782beee85ee5036cc2b2f Mon Sep 17 00:00:00 2001 From: Viktor Kramarenko Date: Tue, 9 Apr 2024 17:21:58 +0300 Subject: [PATCH 19/21] minor fixes Signed-off-by: Viktor Kramarenko --- .../pkg/scheduler/filter.go | 80 +++++++++---------- 1 file changed, 38 insertions(+), 42 deletions(-) diff --git a/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go b/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go index ef59cef4..5865cddf 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go +++ b/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go @@ -630,53 +630,49 @@ func getStorageClassesUsedByPVCs(ctx context.Context, cl client.Client, pvcs map } func getUsedPVC(ctx context.Context, cl client.Client, log logger.Logger, pod *corev1.Pod) (map[string]*corev1.PersistentVolumeClaim, error) { - usedPvc := make(map[string]*corev1.PersistentVolumeClaim, len(pod.Spec.Volumes)) - - var err error - for { - pvcMap, err := getAllPVCsFromNamespace(ctx, cl, pod.Namespace) - if err != nil { - log.Error(err, fmt.Sprintf("[getUsedPVC] unable to get all PVC for Pod %s in the namespace %s", pod.Name, pod.Namespace)) - return nil, err - } + pvcMap, err := getAllPVCsFromNamespace(ctx, cl, pod.Namespace) + if err != nil { + log.Error(err, fmt.Sprintf("[getUsedPVC] unable to get all PVC for Pod %s in the namespace %s", pod.Name, pod.Namespace)) + return nil, err + } - for pvcName := range pvcMap { - log.Trace(fmt.Sprintf("[getUsedPVC] PVC %s is in namespace %s", pod.Namespace, pvcName)) - } + for pvcName := range pvcMap { + log.Trace(fmt.Sprintf("[getUsedPVC] PVC %s is in namespace %s", pod.Namespace, pvcName)) + } - for _, volume := range pod.Spec.Volumes { - if volume.PersistentVolumeClaim != nil { - log.Trace(fmt.Sprintf("[getUsedPVC] Pod %s/%s uses PVC %s", pod.Namespace, pod.Name, volume.PersistentVolumeClaim.ClaimName)) - usedPvc[volume.PersistentVolumeClaim.ClaimName] = pvcMap[volume.PersistentVolumeClaim.ClaimName] - } + usedPvc := make(map[string]*corev1.PersistentVolumeClaim, len(pod.Spec.Volumes)) + for _, volume := range pod.Spec.Volumes { + if volume.PersistentVolumeClaim != nil { + log.Trace(fmt.Sprintf("[getUsedPVC] Pod %s/%s uses PVC %s", pod.Namespace, pod.Name, volume.PersistentVolumeClaim.ClaimName)) + usedPvc[volume.PersistentVolumeClaim.ClaimName] = pvcMap[volume.PersistentVolumeClaim.ClaimName] } - - //filled := false - //if len(pvcMap) > 0 { - // filled = true - // for _, volume := range pod.Spec.Volumes { - // if volume.PersistentVolumeClaim != nil { - // if _, added := usedPvc[volume.PersistentVolumeClaim.ClaimName]; !added { - // filled = false - // log.Warning(fmt.Sprintf("[getUsedPVC] PVC %s was not found in the cache for Pod %s/%s", volume.PersistentVolumeClaim.ClaimName, pod.Namespace, pod.Name)) - // break - // } - // } - // } - //} - // - //if !filled { - // log.Warning(fmt.Sprintf("[getUsedPVC] some PVCs were not found in the cache for Pod %s/%s. Retry to find them again.", pod.Namespace, pod.Name)) - // time.Sleep(100 * time.Millisecond) - // continue - //} - // - //if filled { - // log.Debug(fmt.Sprintf("[getUsedPVC] Every PVC for Pod %s/%s was found in the cache", pod.Namespace, pod.Name)) - // break - //} } + //filled := false + //if len(pvcMap) > 0 { + // filled = true + // for _, volume := range pod.Spec.Volumes { + // if volume.PersistentVolumeClaim != nil { + // if _, added := usedPvc[volume.PersistentVolumeClaim.ClaimName]; !added { + // filled = false + // log.Warning(fmt.Sprintf("[getUsedPVC] PVC %s was not found in the cache for Pod %s/%s", volume.PersistentVolumeClaim.ClaimName, pod.Namespace, pod.Name)) + // break + // } + // } + // } + //} + // + //if !filled { + // log.Warning(fmt.Sprintf("[getUsedPVC] some PVCs were not found in the cache for Pod %s/%s. Retry to find them again.", pod.Namespace, pod.Name)) + // time.Sleep(100 * time.Millisecond) + // continue + //} + // + //if filled { + // log.Debug(fmt.Sprintf("[getUsedPVC] Every PVC for Pod %s/%s was found in the cache", pod.Namespace, pod.Name)) + // break + //} + return usedPvc, err } From ce28b1c51f21323ae70898f6402ab1f2642c040d Mon Sep 17 00:00:00 2001 From: Viktor Kramarenko Date: Tue, 9 Apr 2024 20:34:46 +0300 Subject: [PATCH 20/21] finalizing Signed-off-by: Viktor Kramarenko --- .../pkg/cache/cache.go | 36 ++++---- .../pkg/controller/lvg_watcher_cache.go | 10 +-- .../pkg/controller/pvc_watcher_cache.go | 10 +-- .../pkg/logger/logger.go | 6 ++ .../pkg/scheduler/filter.go | 41 +++------ .../pkg/scheduler/prioritize.go | 8 +- .../pkg/scheduler/route.go | 84 ++++++++----------- 7 files changed, 86 insertions(+), 109 deletions(-) diff --git a/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go b/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go index 244ab037..a6dcb7b8 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go +++ b/images/sds-local-volume-scheduler-extender/pkg/cache/cache.go @@ -111,7 +111,7 @@ func (c *Cache) GetLVGReservedSpace(lvgName string) (int64, error) { var space int64 lvg.(*lvgCache).pvcs.Range(func(pvcName, pvcCh any) bool { - space = pvcCh.(*pvcCache).pvc.Spec.Resources.Requests.Storage().Value() + space += pvcCh.(*pvcCache).pvc.Spec.Resources.Requests.Storage().Value() return true }) @@ -361,10 +361,10 @@ func (c *Cache) RemoveSpaceReservationForPVCWithSelectedNode(pvc *v1.PersistentV selectedNode := pvcCh.(*pvcCache).selectedNode if selectedNode == "" { lvgCh.(*lvgCache).pvcs.Delete(pvcKey) - c.log.Debug(fmt.Sprintf("[RemoveSpaceReservationForPVCWithSelectedNode] removed space reservation for PVC %s in the LVMVolumeGroup %s due the PVC got selected to the node %s", pvcKey, lvgName, selectedNode)) + c.log.Debug(fmt.Sprintf("[RemoveSpaceReservationForPVCWithSelectedNode] removed space reservation for PVC %s in the LVMVolumeGroup %s due the PVC got selected to the node %s", pvcKey, lvgName, pvc.Annotations[SelectedNodeAnnotation])) } else { selectedLVGName = lvgName - c.log.Debug(fmt.Sprintf("[RemoveSpaceReservationForPVCWithSelectedNode] PVC %s got selected to the node %s. It should not be revomed from the LVMVolumeGroup %s", pvcKey, selectedNode, lvgName)) + c.log.Debug(fmt.Sprintf("[RemoveSpaceReservationForPVCWithSelectedNode] PVC %s got selected to the node %s. It should not be revomed from the LVMVolumeGroup %s", pvcKey, pvc.Annotations[SelectedNodeAnnotation], lvgName)) } } c.log.Debug(fmt.Sprintf("[RemoveSpaceReservationForPVCWithSelectedNode] PVC %s space reservation has been removed from LVMVolumeGroup cache", pvcKey)) @@ -434,45 +434,45 @@ func (c *Cache) FindLVGForPVCBySelectedNode(pvc *v1.PersistentVolumeClaim, nodeN return targetLVG } -func (c *Cache) PrintTheCacheTraceLog() { - c.log.Trace("*******************CACHE BEGIN*******************") - c.log.Trace("[LVMVolumeGroups BEGIN]") +func (c *Cache) PrintTheCacheLog() { + c.log.Cache("*******************CACHE BEGIN*******************") + c.log.Cache("[LVMVolumeGroups BEGIN]") c.lvgs.Range(func(lvgName, lvgCh any) bool { - c.log.Trace(fmt.Sprintf("[%s]", lvgName)) + c.log.Cache(fmt.Sprintf("[%s]", lvgName)) lvgCh.(*lvgCache).pvcs.Range(func(pvcName, pvcCh any) bool { - c.log.Trace(fmt.Sprintf(" PVC %s, selected node: %s", pvcName, pvcCh.(*pvcCache).selectedNode)) + c.log.Cache(fmt.Sprintf(" PVC %s, selected node: %s", pvcName, pvcCh.(*pvcCache).selectedNode)) return true }) return true }) - c.log.Trace("[LVMVolumeGroups ENDS]") - c.log.Trace("[PVC and LVG BEGINS]") + c.log.Cache("[LVMVolumeGroups ENDS]") + c.log.Cache("[PVC and LVG BEGINS]") c.pvcLVGs.Range(func(pvcName, lvgs any) bool { - c.log.Trace(fmt.Sprintf("[PVC: %s]", pvcName)) + c.log.Cache(fmt.Sprintf("[PVC: %s]", pvcName)) for _, lvgName := range lvgs.([]string) { - c.log.Trace(fmt.Sprintf(" LVMVolumeGroup: %s", lvgName)) + c.log.Cache(fmt.Sprintf(" LVMVolumeGroup: %s", lvgName)) } return true }) - c.log.Trace("[PVC and LVG ENDS]") - c.log.Trace("[Node and LVG BEGINS]") + c.log.Cache("[PVC and LVG ENDS]") + c.log.Cache("[Node and LVG BEGINS]") c.nodeLVGs.Range(func(nodeName, lvgs any) bool { - c.log.Trace(fmt.Sprintf("[Node: %s]", nodeName)) + c.log.Cache(fmt.Sprintf("[Node: %s]", nodeName)) for _, lvgName := range lvgs.([]string) { - c.log.Trace(fmt.Sprintf(" LVMVolumeGroup name: %s", lvgName)) + c.log.Cache(fmt.Sprintf(" LVMVolumeGroup name: %s", lvgName)) } return true }) - c.log.Trace("[Node and LVG ENDS]") - c.log.Trace("*******************CACHE END*******************") + c.log.Cache("[Node and LVG ENDS]") + c.log.Cache("*******************CACHE END*******************") } func configurePVCKey(pvc *v1.PersistentVolumeClaim) string { diff --git a/images/sds-local-volume-scheduler-extender/pkg/controller/lvg_watcher_cache.go b/images/sds-local-volume-scheduler-extender/pkg/controller/lvg_watcher_cache.go index ec028f75..cc79ddf9 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/controller/lvg_watcher_cache.go +++ b/images/sds-local-volume-scheduler-extender/pkg/controller/lvg_watcher_cache.go @@ -80,11 +80,11 @@ func RunLVGWatcherCacheController( log.Trace(fmt.Sprintf("[RunLVGWatcherCacheController] cached PVC %s/%s belongs to LVMVolumeGroup %s", pvc.Namespace, pvc.Name, lvg.Name)) if pvc.Status.Phase == v1.ClaimBound { log.Trace(fmt.Sprintf("[RunLVGWatcherCacheController] cached PVC %s/%s has Status.Phase Bound. It will be removed from the cache for LVMVolumeGroup %s", pvc.Namespace, pvc.Name, lvg.Name)) - err = cache.RemoveBoundedPVCSpaceReservation(lvg.Name, pvc) - if err != nil { - log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to remove PVC %s/%s from the cache for the LVMVolumeGroup %s", pvc.Namespace, pvc.Name, lvg.Name)) - continue - } + cache.RemovePVCSpaceReservationForced(pvc) + //if err != nil { + // log.Error(err, fmt.Sprintf("[RunLVGWatcherCacheController] unable to remove PVC %s/%s from the cache for the LVMVolumeGroup %s", pvc.Namespace, pvc.Name, lvg.Name)) + // continue + //} log.Debug(fmt.Sprintf("[RunLVGWatcherCacheController] PVC %s/%s was removed from the cache for LVMVolumeGroup %s", pvc.Namespace, pvc.Name, lvg.Name)) } diff --git a/images/sds-local-volume-scheduler-extender/pkg/controller/pvc_watcher_cache.go b/images/sds-local-volume-scheduler-extender/pkg/controller/pvc_watcher_cache.go index a8dd0864..9d5cdda3 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/controller/pvc_watcher_cache.go +++ b/images/sds-local-volume-scheduler-extender/pkg/controller/pvc_watcher_cache.go @@ -113,7 +113,7 @@ func RunPVCWatcherCacheController( } log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] successfully updated PVC %s/%s in the cache", pvc.Namespace, pvc.Name)) - schedulerCache.PrintTheCacheTraceLog() + schedulerCache.PrintTheCacheLog() log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] starts to remove space reservation for PVC %s/%s with selected node from the cache", pvc.Namespace, pvc.Name)) err = schedulerCache.RemoveSpaceReservationForPVCWithSelectedNode(pvc) if err != nil { @@ -121,7 +121,7 @@ func RunPVCWatcherCacheController( return } log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] successfully removed space reservation for PVC %s/%s with selected node", pvc.Namespace, pvc.Name)) - schedulerCache.PrintTheCacheTraceLog() + schedulerCache.PrintTheCacheLog() log.Info("[RunPVCWatcherCacheController] CreateFunc reconciliation ends") }, @@ -194,15 +194,15 @@ func RunPVCWatcherCacheController( } log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] successfully updated PVC %s/%s in the cache", pvc.Namespace, pvc.Name)) - schedulerCache.PrintTheCacheTraceLog() - log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] starts to remove space reservation for PVC %s/%s with selected node from the cache", pvc.Namespace, pvc.Name)) + schedulerCache.PrintTheCacheLog() + log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] starts to remove space reservation for PVC %s/%s with selected node %s from the cache", pvc.Namespace, pvc.Name, pvc.Annotations[cache.SelectedNodeAnnotation])) err = schedulerCache.RemoveSpaceReservationForPVCWithSelectedNode(pvc) if err != nil { log.Error(err, fmt.Sprintf("[RunPVCWatcherCacheController] unable to remove PVC %s/%s space reservation in the cache", pvc.Namespace, pvc.Name)) return } log.Debug(fmt.Sprintf("[RunPVCWatcherCacheController] successfully removed space reservation for PVC %s/%s with selected node", pvc.Namespace, pvc.Name)) - schedulerCache.PrintTheCacheTraceLog() + schedulerCache.PrintTheCacheLog() log.Info("[RunPVCWatcherCacheController] Update Func reconciliation ends") }, diff --git a/images/sds-local-volume-scheduler-extender/pkg/logger/logger.go b/images/sds-local-volume-scheduler-extender/pkg/logger/logger.go index 19dd40a3..0f6bc2de 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/logger/logger.go +++ b/images/sds-local-volume-scheduler-extender/pkg/logger/logger.go @@ -27,6 +27,7 @@ const ( InfoLevel Verbosity = "2" DebugLevel Verbosity = "3" TraceLevel Verbosity = "4" + CacheLevel Verbosity = "5" ) const ( @@ -34,6 +35,7 @@ const ( infoLvl debugLvl traceLvl + cacheLvl ) type ( @@ -79,3 +81,7 @@ func (l Logger) Debug(message string, keysAndValues ...interface{}) { func (l Logger) Trace(message string, keysAndValues ...interface{}) { l.log.V(traceLvl).Info(fmt.Sprintf("TRACE %s", message), keysAndValues...) } + +func (l Logger) Cache(message string, keysAndValues ...interface{}) { + l.log.V(cacheLvl).Info(fmt.Sprintf("CACHE %s", message), keysAndValues...) +} diff --git a/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go b/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go index 5865cddf..b8a0adc9 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go +++ b/images/sds-local-volume-scheduler-extender/pkg/scheduler/filter.go @@ -112,7 +112,7 @@ func (s *scheduler) filter(w http.ResponseWriter, r *http.Request) { s.log.Debug(fmt.Sprintf("[filter] starts to populate the cache for a Pod %s/%s", input.Pod.Namespace, input.Pod.Name)) s.log.Trace(fmt.Sprintf("[filter] cache before the PVC reservation for a Pod %s/%s", input.Pod.Namespace, input.Pod.Name)) - s.cache.PrintTheCacheTraceLog() + s.cache.PrintTheCacheLog() err = populateCache(s.log, filteredNodes.Nodes.Items, input.Pod, s.cache, pvcs, scs) if err != nil { s.log.Error(err, "[filter] unable to populate cache") @@ -121,7 +121,7 @@ func (s *scheduler) filter(w http.ResponseWriter, r *http.Request) { } s.log.Debug(fmt.Sprintf("[filter] successfully populated the cache for a Pod %s/%s", input.Pod.Namespace, input.Pod.Name)) s.log.Trace(fmt.Sprintf("[filter] cache after the PVC reservation for a Pod %s/%s", input.Pod.Namespace, input.Pod.Name)) - s.cache.PrintTheCacheTraceLog() + s.cache.PrintTheCacheLog() w.Header().Set("content-type", "application/json") err = json.NewEncoder(w).Encode(filteredNodes) @@ -636,7 +636,8 @@ func getUsedPVC(ctx context.Context, cl client.Client, log logger.Logger, pod *c return nil, err } - for pvcName := range pvcMap { + for pvcName, pvc := range pvcMap { + log.Trace(fmt.Sprintf("KEY NAME: %s, VALUE NAME: %s", pvcName, pvc.Name)) log.Trace(fmt.Sprintf("[getUsedPVC] PVC %s is in namespace %s", pod.Namespace, pvcName)) } @@ -644,48 +645,26 @@ func getUsedPVC(ctx context.Context, cl client.Client, log logger.Logger, pod *c for _, volume := range pod.Spec.Volumes { if volume.PersistentVolumeClaim != nil { log.Trace(fmt.Sprintf("[getUsedPVC] Pod %s/%s uses PVC %s", pod.Namespace, pod.Name, volume.PersistentVolumeClaim.ClaimName)) - usedPvc[volume.PersistentVolumeClaim.ClaimName] = pvcMap[volume.PersistentVolumeClaim.ClaimName] + pvc := pvcMap[volume.PersistentVolumeClaim.ClaimName] + usedPvc[volume.PersistentVolumeClaim.ClaimName] = &pvc } } - //filled := false - //if len(pvcMap) > 0 { - // filled = true - // for _, volume := range pod.Spec.Volumes { - // if volume.PersistentVolumeClaim != nil { - // if _, added := usedPvc[volume.PersistentVolumeClaim.ClaimName]; !added { - // filled = false - // log.Warning(fmt.Sprintf("[getUsedPVC] PVC %s was not found in the cache for Pod %s/%s", volume.PersistentVolumeClaim.ClaimName, pod.Namespace, pod.Name)) - // break - // } - // } - // } - //} - // - //if !filled { - // log.Warning(fmt.Sprintf("[getUsedPVC] some PVCs were not found in the cache for Pod %s/%s. Retry to find them again.", pod.Namespace, pod.Name)) - // time.Sleep(100 * time.Millisecond) - // continue - //} - // - //if filled { - // log.Debug(fmt.Sprintf("[getUsedPVC] Every PVC for Pod %s/%s was found in the cache", pod.Namespace, pod.Name)) - // break - //} + log.Trace(fmt.Sprintf("[getUsedPVC] HERE Pod %s/%s uses PVC: %v", pod.Namespace, pod.Name, usedPvc)) return usedPvc, err } -func getAllPVCsFromNamespace(ctx context.Context, cl client.Client, namespace string) (map[string]*corev1.PersistentVolumeClaim, error) { +func getAllPVCsFromNamespace(ctx context.Context, cl client.Client, namespace string) (map[string]corev1.PersistentVolumeClaim, error) { list := &corev1.PersistentVolumeClaimList{} err := cl.List(ctx, list, &client.ListOptions{Namespace: namespace}) if err != nil { return nil, err } - pvcs := make(map[string]*corev1.PersistentVolumeClaim, len(list.Items)) + pvcs := make(map[string]corev1.PersistentVolumeClaim, len(list.Items)) for _, pvc := range list.Items { - pvcs[pvc.Name] = &pvc + pvcs[pvc.Name] = pvc } return pvcs, nil diff --git a/images/sds-local-volume-scheduler-extender/pkg/scheduler/prioritize.go b/images/sds-local-volume-scheduler-extender/pkg/scheduler/prioritize.go index 9e09b0bf..50e4a716 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/scheduler/prioritize.go +++ b/images/sds-local-volume-scheduler-extender/pkg/scheduler/prioritize.go @@ -46,7 +46,7 @@ func (s *scheduler) prioritize(w http.ResponseWriter, r *http.Request) { pvcs, err := getUsedPVC(s.ctx, s.client, s.log, input.Pod) if err != nil { - s.log.Error(err, "[prioritize] unable to get PVC from the Pod") + s.log.Error(err, fmt.Sprintf("[prioritize] unable to get PVC from the Pod %s/%s", input.Pod.Namespace, input.Pod.Name)) http.Error(w, "internal server error", http.StatusInternalServerError) return } @@ -61,7 +61,7 @@ func (s *scheduler) prioritize(w http.ResponseWriter, r *http.Request) { scs, err := getStorageClassesUsedByPVCs(s.ctx, s.client, pvcs) if err != nil { - s.log.Error(err, "[prioritize] unable to get StorageClasses from the PVC") + s.log.Error(err, fmt.Sprintf("[prioritize] unable to get StorageClasses from the PVC for Pod %s/%s", input.Pod.Namespace, input.Pod.Name)) http.Error(w, "bad request", http.StatusBadRequest) return } @@ -72,7 +72,7 @@ func (s *scheduler) prioritize(w http.ResponseWriter, r *http.Request) { s.log.Debug(fmt.Sprintf("[prioritize] starts to extract pvcRequests size for Pod %s/%s", input.Pod.Namespace, input.Pod.Name)) pvcRequests, err := extractRequestedSize(s.ctx, s.client, s.log, pvcs, scs) if err != nil { - s.log.Error(err, fmt.Sprintf("[filter] unable to extract request size for a pod %s", input.Pod.Name)) + s.log.Error(err, fmt.Sprintf("[filter] unable to extract request size for Pod %s/%s", input.Pod.Namespace, input.Pod.Name)) http.Error(w, "bad request", http.StatusBadRequest) } s.log.Debug(fmt.Sprintf("[filter] successfully extracted the pvcRequests size for Pod %s/%s", input.Pod.Namespace, input.Pod.Name)) @@ -188,10 +188,12 @@ func scoreNodes( errs <- err return } + log.Trace(fmt.Sprintf("[scoreNodes] LVMVolumeGroup %s total size: %s", lvg.Name, lvgTotalSize.String())) totalFreeSpaceLeft += getFreeSpaceLeftPercent(freeSpace.Value(), pvcReq.RequestedSize, lvgTotalSize.Value()) } averageFreeSpace := totalFreeSpaceLeft / int64(len(pvcs)) + log.Trace(fmt.Sprintf("[scoreNodes] average free space left for the node: %s", node.Name)) score := getNodeScore(averageFreeSpace, divisor) log.Trace(fmt.Sprintf("[scoreNodes] node %s has score %d with average free space left (after all PVC bounded), percent %d", node.Name, score, averageFreeSpace)) diff --git a/images/sds-local-volume-scheduler-extender/pkg/scheduler/route.go b/images/sds-local-volume-scheduler-extender/pkg/scheduler/route.go index b40df467..f97d29a8 100644 --- a/images/sds-local-volume-scheduler-extender/pkg/scheduler/route.go +++ b/images/sds-local-volume-scheduler-extender/pkg/scheduler/route.go @@ -57,10 +57,6 @@ func (s *scheduler) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.log.Debug("[ServeHTTP] stat route starts handling the request") s.getCacheStat(w, r) s.log.Debug("[ServeHTTP] stat route ends handling the request") - case "/reserved": - s.log.Debug("[ServeHTTP] reserved route starts handling the request") - s.getReservedSpace(w, r) - s.log.Debug("[ServeHTTP] reserved route ends handling the request") default: http.Error(w, "not found", http.StatusNotFound) } @@ -88,27 +84,27 @@ func status(w http.ResponseWriter, r *http.Request) { func (s *scheduler) getCache(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - s.cache.PrintTheCacheTraceLog() + s.cache.PrintTheCacheLog() result := make(map[string][]struct { - pvcName string - nodeName string + pvcName string + selectedNode string + status string + size string }) lvgs := s.cache.GetAllLVG() - //s.log.Info(fmt.Sprintf("LVG from cache: %v", lvgs)) for _, lvg := range lvgs { pvcs, err := s.cache.GetAllPVCForLVG(lvg.Name) if err != nil { s.log.Error(err, "something bad") } - //for _, pvc := range pvcs { - // s.log.Trace(fmt.Sprintf("LVG %s has PVC from cache: %v", lvg, pvc.Name)) - //} result[lvg.Name] = make([]struct { - pvcName string - nodeName string + pvcName string + selectedNode string + status string + size string }, 0) for _, pvc := range pvcs { @@ -117,17 +113,37 @@ func (s *scheduler) getCache(w http.ResponseWriter, r *http.Request) { s.log.Error(err, "something bad") } result[lvg.Name] = append(result[lvg.Name], struct { - pvcName string - nodeName string - }{pvcName: pvc.Name, nodeName: selectedNode}) + pvcName string + selectedNode string + status string + size string + }{pvcName: pvc.Name, selectedNode: selectedNode, status: string(pvc.Status.Phase), size: pvc.Spec.Resources.Requests.Storage().String()}) } } - //s.log.Info(fmt.Sprintf("Result len: %d", len(result))) - _, err := w.Write([]byte(fmt.Sprintf("%+v", result))) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte("unable to write the cache")) + for lvgName, pvcs := range result { + reserved, err := s.cache.GetLVGReservedSpace(lvgName) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("unable to write the cache")) + } + + _, err = w.Write([]byte(fmt.Sprintf("LVMVolumeGroup: %s Reserved: %s\n", lvgName, resource.NewQuantity(reserved, resource.BinarySI)))) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("unable to write the cache")) + } + + for _, pvc := range pvcs { + _, err = w.Write([]byte(fmt.Sprintf("\tPVC: %s\n", pvc.pvcName))) + _, err = w.Write([]byte(fmt.Sprintf("\t\tNodeName: %s\n", pvc.selectedNode))) + _, err = w.Write([]byte(fmt.Sprintf("\t\tStatus: %s\n", pvc.status))) + _, err = w.Write([]byte(fmt.Sprintf("\t\tSize: %s\n", pvc.size))) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("unable to write the cache")) + } + } } } @@ -151,29 +167,3 @@ func (s *scheduler) getCacheStat(w http.ResponseWriter, r *http.Request) { w.Write([]byte("unable to write the cache")) } } - -func (s *scheduler) getReservedSpace(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - - lvgs := s.cache.GetAllLVG() - - result := make(map[string]int64, len(lvgs)) - for _, lvg := range lvgs { - space, err := s.cache.GetLVGReservedSpace(lvg.Name) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte("unable to get the space")) - } - - result[lvg.Name] = space - } - - for lvgName, space := range result { - _, err := w.Write([]byte(fmt.Sprintf("LVMVolumeGroup: %s, Reserved space: %s\n", lvgName, resource.NewQuantity(space, resource.BinarySI)))) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte("unable to write the cache")) - } - } - -} From 17e27063ad91c72d8e308bd1c4bac81e8b47eabe Mon Sep 17 00:00:00 2001 From: Aleksandr Zimin Date: Wed, 10 Apr 2024 00:43:06 +0300 Subject: [PATCH 21/21] increase workers for csi controller from 1 to 10 Signed-off-by: Aleksandr Zimin --- templates/sds-local-volume-csi/controller.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/sds-local-volume-csi/controller.yaml b/templates/sds-local-volume-csi/controller.yaml index e405eff3..ef7b0c1c 100644 --- a/templates/sds-local-volume-csi/controller.yaml +++ b/templates/sds-local-volume-csi/controller.yaml @@ -370,7 +370,7 @@ spec: - --timeout=1m - --leader-election=true - --leader-election-namespace=$(NAMESPACE) - - --worker-threads=1 + - --worker-threads=10 env: - name: ADDRESS value: /csi/csi.sock @@ -424,7 +424,7 @@ spec: - --enable-capacity - --extra-create-metadata - --capacity-ownerref-level=2 - - --worker-threads=1 + - --worker-threads=10 env: - name: ADDRESS value: /csi/csi.sock @@ -458,7 +458,7 @@ spec: - --csi-address=$(ADDRESS) - --leader-election=true - --leader-election-namespace=$(NAMESPACE) - - --worker-threads=1 + - --worker-threads=10 env: - name: ADDRESS value: /csi/csi.sock @@ -489,7 +489,7 @@ spec: - --handle-volume-inuse-error=false - --leader-election=true - --leader-election-namespace=$(NAMESPACE) - - --workers=1 + - --workers=10 env: - name: ADDRESS value: /csi/csi.sock