[controller] Fix the cache freezed Bound PVC

images/sds-local-volume-controller/src/go.mod

@@ -4,7 +4,7 @@ go 1.22.2
 
 require (
 	github.com/deckhouse/sds-local-volume/api v0.0.0-20240816081122-3de604d3d889
-	github.com/deckhouse/sds-node-configurator/api v0.0.0-20240925090458-249de2896583
+	github.com/deckhouse/sds-node-configurator/api v0.0.0-20240926063625-6815fd9556ea
 	github.com/go-logr/logr v1.4.2
 	github.com/onsi/ginkgo/v2 v2.20.0
 	github.com/onsi/gomega v1.34.1

images/sds-local-volume-controller/src/go.sum

@@ -14,6 +14,8 @@ github.com/deckhouse/sds-node-configurator/api v0.0.0-20240919102704-a035b4a92e7
 github.com/deckhouse/sds-node-configurator/api v0.0.0-20240919102704-a035b4a92e77/go.mod h1:H71+9G0Jr46Qs0BA3z3/xt0h9lbnJnCEYcaCJCWFBf0=
 github.com/deckhouse/sds-node-configurator/api v0.0.0-20240925090458-249de2896583 h1:HQd5YFQqoHj/CQwBKFCyuVCQmNV0PdML8QJiyDka4fQ=
 github.com/deckhouse/sds-node-configurator/api v0.0.0-20240925090458-249de2896583/go.mod h1:H71+9G0Jr46Qs0BA3z3/xt0h9lbnJnCEYcaCJCWFBf0=
+github.com/deckhouse/sds-node-configurator/api v0.0.0-20240926063625-6815fd9556ea h1:RIbBqkoLvWHFNIpmq5LYObcwpRNWAEE6itzvwi/bvEQ=
+github.com/deckhouse/sds-node-configurator/api v0.0.0-20240926063625-6815fd9556ea/go.mod h1:H71+9G0Jr46Qs0BA3z3/xt0h9lbnJnCEYcaCJCWFBf0=
 github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU=
 github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84=

images/sds-local-volume-csi/src/go.mod

@@ -5,7 +5,7 @@ go 1.22.3
 require (
 	github.com/container-storage-interface/spec v1.10.0
 	github.com/deckhouse/sds-local-volume/api v0.0.0-20240903071950-ed9d3bba999b
-	github.com/deckhouse/sds-node-configurator/api v0.0.0-20240925090458-249de2896583
+	github.com/deckhouse/sds-node-configurator/api v0.0.0-20240926063625-6815fd9556ea
 	github.com/go-logr/logr v1.4.2
 	github.com/golang/protobuf v1.5.4
 	github.com/google/uuid v1.6.0

images/sds-local-volume-csi/src/go.sum

@@ -12,6 +12,8 @@ github.com/deckhouse/sds-node-configurator/api v0.0.0-20240919102704-a035b4a92e7
 github.com/deckhouse/sds-node-configurator/api v0.0.0-20240919102704-a035b4a92e77/go.mod h1:H71+9G0Jr46Qs0BA3z3/xt0h9lbnJnCEYcaCJCWFBf0=
 github.com/deckhouse/sds-node-configurator/api v0.0.0-20240925090458-249de2896583 h1:HQd5YFQqoHj/CQwBKFCyuVCQmNV0PdML8QJiyDka4fQ=
 github.com/deckhouse/sds-node-configurator/api v0.0.0-20240925090458-249de2896583/go.mod h1:H71+9G0Jr46Qs0BA3z3/xt0h9lbnJnCEYcaCJCWFBf0=
+github.com/deckhouse/sds-node-configurator/api v0.0.0-20240926063625-6815fd9556ea h1:RIbBqkoLvWHFNIpmq5LYObcwpRNWAEE6itzvwi/bvEQ=
+github.com/deckhouse/sds-node-configurator/api v0.0.0-20240926063625-6815fd9556ea/go.mod h1:H71+9G0Jr46Qs0BA3z3/xt0h9lbnJnCEYcaCJCWFBf0=
 github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU=
 github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=

images/sds-local-volume-scheduler-extender/src/cmd/main.go

@@ -61,6 +61,7 @@ type Config struct {
 	HealthProbeBindAddress string  `json:"health-probe-bind-address"`
 	CertFile               string  `json:"cert-file"`
 	KeyFile                string  `json:"key-file"`
+	PVCExpiredDurationSec  int     `json:"pvc-expired-duration-sec"`
 }
 
 var cfgFilePath string
@@ -73,12 +74,13 @@ var resourcesSchemeFuncs = []func(*runtime.Scheme) error{
 }
 
 var config = &Config{
-	ListenAddr:     defaultListenAddr,
-	DefaultDivisor: defaultDivisor,
-	LogLevel:       "2",
-	CacheSize:      defaultCacheSize,
-	CertFile:       defaultcertFile,
-	KeyFile:        defaultkeyFile,
+	ListenAddr:            defaultListenAddr,
+	DefaultDivisor:        defaultDivisor,
+	LogLevel:              "2",
+	CacheSize:             defaultCacheSize,
+	CertFile:              defaultcertFile,
+	KeyFile:               defaultkeyFile,
+	PVCExpiredDurationSec: cache.DefaultPVCExpiredDurationSec,
 }
 
 var rootCmd = &cobra.Command{
@@ -167,7 +169,7 @@ func subMain(ctx context.Context) error {
 		return err
 	}
 
-	schedulerCache := cache.NewCache(*log)
+	schedulerCache := cache.NewCache(*log, config.PVCExpiredDurationSec)
 	log.Info("[subMain] scheduler cache was initialized")
 
 	h, err := scheduler.NewHandler(ctx, mgr.GetClient(), *log, schedulerCache, config.DefaultDivisor)

images/sds-local-volume-scheduler-extender/src/go.mod

@@ -4,7 +4,7 @@ go 1.22.2
 
 require (
 	github.com/deckhouse/sds-local-volume/api v0.0.0-20240903071950-ed9d3bba999b
-	github.com/deckhouse/sds-node-configurator/api v0.0.0-20240925090458-249de2896583
+	github.com/deckhouse/sds-node-configurator/api v0.0.0-20240926063625-6815fd9556ea
 	github.com/go-logr/logr v1.4.2
 	github.com/go-logr/zapr v1.3.0
 	github.com/spf13/cobra v1.8.1

images/sds-local-volume-scheduler-extender/src/go.sum

@@ -15,6 +15,8 @@ github.com/deckhouse/sds-node-configurator/api v0.0.0-20240919102704-a035b4a92e7
 github.com/deckhouse/sds-node-configurator/api v0.0.0-20240919102704-a035b4a92e77/go.mod h1:H71+9G0Jr46Qs0BA3z3/xt0h9lbnJnCEYcaCJCWFBf0=
 github.com/deckhouse/sds-node-configurator/api v0.0.0-20240925090458-249de2896583 h1:HQd5YFQqoHj/CQwBKFCyuVCQmNV0PdML8QJiyDka4fQ=
 github.com/deckhouse/sds-node-configurator/api v0.0.0-20240925090458-249de2896583/go.mod h1:H71+9G0Jr46Qs0BA3z3/xt0h9lbnJnCEYcaCJCWFBf0=
+github.com/deckhouse/sds-node-configurator/api v0.0.0-20240926063625-6815fd9556ea h1:RIbBqkoLvWHFNIpmq5LYObcwpRNWAEE6itzvwi/bvEQ=
+github.com/deckhouse/sds-node-configurator/api v0.0.0-20240926063625-6815fd9556ea/go.mod h1:H71+9G0Jr46Qs0BA3z3/xt0h9lbnJnCEYcaCJCWFBf0=
 github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU=
 github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k=

images/sds-local-volume-scheduler-extender/src/pkg/cache/cache.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"sync"
+	"time"
 
 	snc "github.com/deckhouse/sds-node-configurator/api/v1alpha1"
 	v1 "k8s.io/api/core/v1"
@@ -14,17 +15,20 @@ import (
 )
 
 const (
+	DefaultPVCExpiredDurationSec = 30
+
 	pvcPerLVGCount         = 150
 	lvgsPerPVCCount        = 5
 	lvgsPerNodeCount       = 5
 	SelectedNodeAnnotation = "volume.kubernetes.io/selected-node"
 )
 
 type Cache struct {
-	lvgs     sync.Map // map[string]*lvgCache
-	pvcLVGs  sync.Map // map[string][]string
-	nodeLVGs sync.Map // map[string][]string
-	log      logger.Logger
+	lvgs            sync.Map // map[string]*lvgCache
+	pvcLVGs         sync.Map // map[string][]string
+	nodeLVGs        sync.Map // map[string][]string
+	log             logger.Logger
+	expiredDuration time.Duration
 }
 
 type lvgCache struct {
@@ -43,18 +47,56 @@ type pvcCache struct {
 }
 
 // NewCache initialize new cache.
-func NewCache(logger logger.Logger) *Cache {
-	return &Cache{
-		log: logger,
+func NewCache(logger logger.Logger, pvcExpDurSec int) *Cache {
+	ch := &Cache{
+		log:             logger,
+		expiredDuration: time.Duration(pvcExpDurSec) * time.Second,
 	}
+
+	go func() {
+		timer := time.NewTimer(ch.expiredDuration)
+
+		for range timer.C {
+			ch.clearBoundExpiredPVC()
+			timer.Reset(ch.expiredDuration)
+		}
+	}()
+
+	return ch
+}
+
+func (c *Cache) clearBoundExpiredPVC() {
+	c.log.Debug("[clearBoundExpiredPVC] starts to clear expired PVC")
+	c.lvgs.Range(func(lvgName, _ any) bool {
+		pvcs, err := c.GetAllPVCForLVG(lvgName.(string))
+		if err != nil {
+			c.log.Error(err, fmt.Sprintf("[clearBoundExpiredPVC] unable to get PVCs for the LVMVolumeGroup %s", lvgName.(string)))
+			return false
+		}
+
+		for _, pvc := range pvcs {
+			if pvc.Status.Phase != v1.ClaimBound {
+				c.log.Trace(fmt.Sprintf("[clearBoundExpiredPVC] PVC %s is not in a Bound state", pvc.Name))
+				continue
+			}
+
+			if time.Since(pvc.CreationTimestamp.Time) > c.expiredDuration {
+				c.log.Warning(fmt.Sprintf("[clearBoundExpiredPVC] PVC %s is in a Bound state and expired, remove it from the cache", pvc.Name))
+				c.RemovePVCFromTheCache(pvc)
+			} else {
+				c.log.Trace(fmt.Sprintf("[clearBoundExpiredPVC] PVC %s is in a Bound state but not expired yet.", pvc.Name))
+			}
+		}
+
+		return true
+	})
+	c.log.Debug("[clearBoundExpiredPVC] finished the expired PVC clearing")
 }
 
 // AddLVG adds selected LVMVolumeGroup resource to the cache. If it is already stored, does nothing.
 func (c *Cache) AddLVG(lvg *snc.LVMVolumeGroup) {
 	_, loaded := c.lvgs.LoadOrStore(lvg.Name, &lvgCache{
-		lvg:       lvg,
-		thickPVCs: sync.Map{},
-		thinPools: sync.Map{},
+		lvg: lvg,
 	})
 	if loaded {
 		c.log.Debug(fmt.Sprintf("[AddLVG] the LVMVolumeGroup %s has been already added to the cache", lvg.Name))
@@ -76,29 +118,30 @@ func (c *Cache) AddLVG(lvg *snc.LVMVolumeGroup) {
 
 // UpdateLVG updated selected LVMVolumeGroup resource in the cache. If such LVMVolumeGroup is not stored, returns an error.
 func (c *Cache) UpdateLVG(lvg *snc.LVMVolumeGroup) error {
-	if lvgCh, found := c.lvgs.Load(lvg.Name); found {
-		lvgCh.(*lvgCache).lvg = lvg
-
-		c.log.Trace(fmt.Sprintf("[UpdateLVG] the LVMVolumeGroup %s nodes: %v", lvg.Name, lvg.Status.Nodes))
-		for _, node := range lvg.Status.Nodes {
-			lvgsOnTheNode, _ := c.nodeLVGs.Load(node.Name)
-			if lvgsOnTheNode == nil {
-				lvgsOnTheNode = make([]string, 0, lvgsPerNodeCount)
-			}
+	lvgCh, found := c.lvgs.Load(lvg.Name)
+	if !found {
+		return fmt.Errorf("the LVMVolumeGroup %s was not found in the lvgCh", lvg.Name)
+	}
 
-			if !slices2.Contains(lvgsOnTheNode.([]string), lvg.Name) {
-				lvgsOnTheNode = append(lvgsOnTheNode.([]string), lvg.Name)
-				c.log.Debug(fmt.Sprintf("[UpdateLVG] the LVMVolumeGroup %s has been added to the node %s", lvg.Name, node.Name))
-				c.nodeLVGs.Store(node.Name, lvgsOnTheNode)
-			} else {
-				c.log.Debug(fmt.Sprintf("[UpdateLVG] the LVMVolumeGroup %s has been already added to the node %s", lvg.Name, node.Name))
-			}
+	lvgCh.(*lvgCache).lvg = lvg
+
+	c.log.Trace(fmt.Sprintf("[UpdateLVG] the LVMVolumeGroup %s nodes: %v", lvg.Name, lvg.Status.Nodes))
+	for _, node := range lvg.Status.Nodes {
+		lvgsOnTheNode, _ := c.nodeLVGs.Load(node.Name)
+		if lvgsOnTheNode == nil {
+			lvgsOnTheNode = make([]string, 0, lvgsPerNodeCount)
 		}
 
-		return nil
+		if !slices2.Contains(lvgsOnTheNode.([]string), lvg.Name) {
+			lvgsOnTheNode = append(lvgsOnTheNode.([]string), lvg.Name)
+			c.log.Debug(fmt.Sprintf("[UpdateLVG] the LVMVolumeGroup %s has been added to the node %s", lvg.Name, node.Name))
+			c.nodeLVGs.Store(node.Name, lvgsOnTheNode)
+		} else {
+			c.log.Debug(fmt.Sprintf("[UpdateLVG] the LVMVolumeGroup %s has been already added to the node %s", lvg.Name, node.Name))
+		}
 	}
 
-	return fmt.Errorf("the LVMVolumeGroup %s was not found in the lvgCh", lvg.Name)
+	return nil
 }
 
 // TryGetLVG returns selected LVMVolumeGroup resource if it is stored in the cache, otherwise returns nil.
@@ -186,8 +229,9 @@ func (c *Cache) DeleteLVG(lvgName string) {
 	c.nodeLVGs.Range(func(_, lvgNames any) bool {
 		for i, lvg := range lvgNames.([]string) {
 			if lvg == lvgName {
-				//nolint:gocritic
+				//nolint:gocritic,ineffassign
 				lvgNames = append(lvgNames.([]string)[:i], lvgNames.([]string)[i+1:]...)
+				return false
 			}
 		}
 
@@ -197,8 +241,9 @@ func (c *Cache) DeleteLVG(lvgName string) {
 	c.pvcLVGs.Range(func(_, lvgNames any) bool {
 		for i, lvg := range lvgNames.([]string) {
 			if lvg == lvgName {
-				//nolint:gocritic
+				//nolint:gocritic,ineffassign
 				lvgNames = append(lvgNames.([]string)[:i], lvgNames.([]string)[i+1:]...)
+				return false
 			}
 		}
 
@@ -645,27 +690,24 @@ func (c *Cache) RemoveSpaceReservationForPVCWithSelectedNode(pvc *v1.PersistentV
 
 // RemovePVCFromTheCache completely removes selected PVC in the cache.
 func (c *Cache) RemovePVCFromTheCache(pvc *v1.PersistentVolumeClaim) {
-	targetPvcKey := configurePVCKey(pvc)
-
-	c.log.Debug(fmt.Sprintf("[RemovePVCFromTheCache] run full cache wipe for PVC %s", targetPvcKey))
-	c.pvcLVGs.Range(func(pvcKey, lvgArray any) bool {
-		if pvcKey == targetPvcKey {
-			for _, lvgName := range lvgArray.([]string) {
-				lvgCh, found := c.lvgs.Load(lvgName)
-				if found {
-					lvgCh.(*lvgCache).thickPVCs.Delete(pvcKey.(string))
-					lvgCh.(*lvgCache).thinPools.Range(func(_, tpCh any) bool {
-						tpCh.(*thinPoolCache).pvcs.Delete(pvcKey)
-						return true
-					})
-				}
+	pvcKey := configurePVCKey(pvc)
+
+	c.log.Debug(fmt.Sprintf("[RemovePVCFromTheCache] run full cache wipe for PVC %s", pvcKey))
+	lvgSlice, ok := c.pvcLVGs.Load(pvcKey)
+	if ok {
+		for _, lvgName := range lvgSlice.([]string) {
+			lvgCh, found := c.lvgs.Load(lvgName)
+			if found {
+				lvgCh.(*lvgCache).thickPVCs.Delete(pvcKey)
+				lvgCh.(*lvgCache).thinPools.Range(func(_, tpCh any) bool {
+					tpCh.(*thinPoolCache).pvcs.Delete(pvcKey)
+					return true
+				})
 			}
 		}
+	}
 
-		return true
-	})
-
-	c.pvcLVGs.Delete(targetPvcKey)
+	c.pvcLVGs.Delete(pvcKey)
 }
 
 // FindLVGForPVCBySelectedNode finds a suitable LVMVolumeGroup resource's name for selected PVC based on selected node. If no such LVMVolumeGroup found, returns empty string.
@@ -688,6 +730,7 @@ func (c *Cache) FindLVGForPVCBySelectedNode(pvc *v1.PersistentVolumeClaim, nodeN
 	for _, lvgName := range lvgsForPVC.([]string) {
 		if slices2.Contains(lvgsOnTheNode.([]string), lvgName) {
 			targetLVG = lvgName
+			break
 		}
 	}