diff --git a/Chart.yaml b/Chart.yaml index aad997a7..773c417f 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -1,2 +1,2 @@ -name: sds-lvm-provisioner +name: sds-lvm version: 0.0.1 diff --git a/images/sds-lvm-scheduler-extender/cmd/cmd/root.go b/images/sds-lvm-scheduler-extender/cmd/cmd/root.go index eac01047..73b4d470 100644 --- a/images/sds-lvm-scheduler-extender/cmd/cmd/root.go +++ b/images/sds-lvm-scheduler-extender/cmd/cmd/root.go @@ -4,62 +4,70 @@ import ( "context" "errors" "fmt" + v1 "k8s.io/api/core/v1" + sv1 "k8s.io/api/storage/v1" + extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/runtime" "net/http" "os" "os/signal" + "sds-lvm-scheduler-extender/pkg/api/v1alpha1" + "sds-lvm-scheduler-extender/pkg/kubutils" + "sds-lvm-scheduler-extender/pkg/logger" + "sds-lvm-scheduler-extender/pkg/scheduler" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sync" "syscall" "time" - "sds-lvm-scheduler-extender/pkg/scheduler" + apiruntime "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" "github.com/spf13/cobra" - "github.com/topolvm/topolvm" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/yaml" ) var cfgFilePath string -var zapOpts zap.Options -const defaultDivisor = 1 -const defaultListenAddr = ":8000" +var resourcesSchemeFuncs = []func(*apiruntime.Scheme) error{ + v1alpha1.AddToScheme, + clientgoscheme.AddToScheme, + extv1.AddToScheme, + v1.AddToScheme, + sv1.AddToScheme, +} + +const ( + defaultDivisor = 1 + defaultListenAddr = ":8000" +) -// Config represents configuration parameters for topolvm-scheduler type Config struct { - // ListenAddr is listen address of topolvm-scheduler. - ListenAddr string `json:"listen"` - // Divisors is a mapping between device-class names and their divisors. - Divisors map[string]float64 `json:"divisors"` - // DefaultDivisor is the default divisor value. + ListenAddr string `json:"listen"` DefaultDivisor float64 `json:"default-divisor"` + LogLevel string `json:"log-level"` } var config = &Config{ ListenAddr: defaultListenAddr, DefaultDivisor: defaultDivisor, + LogLevel: "2", } var rootCmd = &cobra.Command{ - Use: "topolvm-scheduler", - Version: topolvm.Version, - Short: "a scheduler-extender for TopoLVM", - Long: `A scheduler-extender for TopoLVM. - + Use: "sds-lvm-scheduler", + Version: "development", + Short: "a scheduler-extender for SDS-LVM", + Long: `A scheduler-extender for SDS-LVM. The extender implements filter and prioritize verbs. - -The filter verb is "predicate" and served at "/predicate" via HTTP. +The filter verb is "filter" and served at "/filter" via HTTP. It filters out nodes that have less storage capacity than requested. The requested capacity is read from "capacity.topolvm.io/" resource value. - The prioritize verb is "prioritize" and served at "/prioritize" via HTTP. It scores nodes with this formula: - min(10, max(0, log2(capacity >> 30 / divisor))) - The default divisor is 1. It can be changed with a command-line option. `, RunE: func(cmd *cobra.Command, args []string) error { @@ -69,9 +77,6 @@ The default divisor is 1. It can be changed with a command-line option. } func subMain(parentCtx context.Context) error { - ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zapOpts))) - logger := log.FromContext(parentCtx) - if len(cfgFilePath) != 0 { b, err := os.ReadFile(cfgFilePath) if err != nil { @@ -83,32 +88,59 @@ func subMain(parentCtx context.Context) error { } } - h, err := scheduler.NewHandler(config.DefaultDivisor, config.Divisors) + log, err := logger.NewLogger(logger.Verbosity(config.LogLevel)) + if err != nil { + fmt.Println(fmt.Sprintf("[subMain] unable to initialize logger, err: %s", err.Error())) + } + log.Info(fmt.Sprintf("[subMain] logger has been initialized, log level: %s", config.LogLevel)) + ctrl.SetLogger(log.GetLogger()) + + kConfig, err := kubutils.KubernetesDefaultConfigCreate() + if err != nil { + log.Error(err, "[subMain] unable to KubernetesDefaultConfigCreate") + } + log.Info("[subMain] kubernetes config has been successfully created.") + + scheme := runtime.NewScheme() + for _, f := range resourcesSchemeFuncs { + err := f(scheme) + if err != nil { + log.Error(err, "[subMain] unable to add scheme to func") + os.Exit(1) + } + } + log.Info("[subMain] successfully read scheme CR") + + cl, err := client.New(kConfig, client.Options{ + Scheme: scheme, + WarningHandler: client.WarningHandlerOptions{}, + }) + + h, err := scheduler.NewHandler(cl, *log, config.DefaultDivisor) if err != nil { return err } + log.Info("[subMain] scheduler handler initialized") serv := &http.Server{ Addr: config.ListenAddr, Handler: accessLogHandler(parentCtx, h), ReadTimeout: 30 * time.Second, } - var wg sync.WaitGroup defer wg.Wait() - ctx, stop := signal.NotifyContext(parentCtx, os.Interrupt, syscall.SIGTERM) defer stop() // stop() should be called before wg.Wait() to stop the goroutine correctly. - wg.Add(1) go func() { defer wg.Done() <-ctx.Done() if err := serv.Shutdown(parentCtx); err != nil { - logger.Error(err, "failed to shutdown gracefully") + log.Error(err, "failed to shutdown gracefully") } }() + log.Info(fmt.Sprintf("[subMain] starts serving on: %s", config.ListenAddr)) err = serv.ListenAndServe() if !errors.Is(err, http.ErrServerClosed) { return err @@ -124,7 +156,6 @@ func Execute() { os.Exit(1) } } - func init() { rootCmd.PersistentFlags().StringVar(&cfgFilePath, "config", "", "config file") } diff --git a/images/sds-lvm-scheduler-extender/config/config.go b/images/sds-lvm-scheduler-extender/config/config.go new file mode 100644 index 00000000..4fb068fa --- /dev/null +++ b/images/sds-lvm-scheduler-extender/config/config.go @@ -0,0 +1,45 @@ +/* +Copyright 2023 Flant JSC +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "os" + "sds-lvm-scheduler-extender/pkg/logger" +) + +const ( + NodeName = "KUBE_NODE_NAME" + LogLevel = "LOG_LEVEL" +) + +type Options struct { + NodeName string + Version string + Loglevel logger.Verbosity +} + +func NewConfig() *Options { + var opts Options + + loglevel := os.Getenv(LogLevel) + if loglevel == "" { + opts.Loglevel = logger.DebugLevel + } else { + opts.Loglevel = logger.Verbosity(loglevel) + } + + opts.Version = "dev" + + return &opts +} diff --git a/images/sds-lvm-scheduler-extender/go.mod b/images/sds-lvm-scheduler-extender/go.mod index dc1dc24a..b08262b1 100644 --- a/images/sds-lvm-scheduler-extender/go.mod +++ b/images/sds-lvm-scheduler-extender/go.mod @@ -3,12 +3,16 @@ module sds-lvm-scheduler-extender go 1.21 require ( + github.com/go-logr/logr v1.4.1 github.com/go-logr/zapr v1.3.0 github.com/spf13/cobra v1.8.0 github.com/topolvm/topolvm v0.25.0 go.uber.org/zap v1.26.0 k8s.io/api v0.29.1 + k8s.io/apiextensions-apiserver v0.29.0 k8s.io/apimachinery v0.29.1 + k8s.io/client-go v0.29.0 + k8s.io/klog/v2 v2.110.1 sigs.k8s.io/controller-runtime v0.17.0 sigs.k8s.io/yaml v1.4.0 ) @@ -20,7 +24,6 @@ require ( github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch/v5 v5.8.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect @@ -60,10 +63,7 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.29.0 // indirect - k8s.io/client-go v0.29.0 // indirect k8s.io/component-base v0.29.0 // indirect - k8s.io/klog/v2 v2.110.1 // indirect k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/images/sds-lvm-scheduler-extender/go.sum b/images/sds-lvm-scheduler-extender/go.sum index bff474cc..95a93e3d 100644 --- a/images/sds-lvm-scheduler-extender/go.sum +++ b/images/sds-lvm-scheduler-extender/go.sum @@ -1,6 +1,9 @@ github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc= +github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/sprig v2.15.0+incompatible h1:0gSxPGWS9PAr7U2NsQ2YQg6juRDINkUyuvbb4b2Xm8w= +github.com/Masterminds/sprig v2.15.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg= +github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= @@ -13,10 +16,13 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= +github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -31,7 +37,9 @@ github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gobuffalo/flect v1.0.2 h1:eqjPGSo2WmjgY2XlpGwo2NXgL3RucAKo4k4qQMNA5sA= +github.com/gobuffalo/flect v1.0.2/go.mod h1:A5msMlrHtLqh9umBSnvabjsMrCcCpAyzglnDvkbYKHs= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= @@ -50,9 +58,11 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk= +github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -65,6 +75,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -72,7 +83,9 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -83,8 +96,11 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-proto-validators v0.0.0-20180403085117-0950a7990007 h1:28i1IjGcx8AofiB4N3q5Yls55VEaitzuEPkFJEVgGkA= +github.com/mwitkow/go-proto-validators v0.0.0-20180403085117-0950a7990007/go.mod h1:m2XC9Qq0AlmmVksL6FktJCdTYyLk7V3fKyp0sl1yWQo= github.com/onsi/ginkgo/v2 v2.14.0 h1:vSmGj2Z5YPb9JwCWT6z6ihcUvDhuXLc3sJiqd3jMKAY= +github.com/onsi/ginkgo/v2 v2.14.0/go.mod h1:JkUdW7JkN0V6rFvsHcJ478egV3XH9NxpD27Hal/PhZw= github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= +github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -98,8 +114,11 @@ github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGy github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/pseudomuto/protoc-gen-doc v1.5.0 h1:pHZp0MEiT68jrZV8js8BS7E9ZEnlSLegoQbbtXj5lfo= +github.com/pseudomuto/protoc-gen-doc v1.5.0/go.mod h1:exDTOVwqpp30eV/EDPFLZy3Pwr2sn6hBC1WIYH/UbIg= github.com/pseudomuto/protokit v0.2.0 h1:hlnBDcy3YEDXH7kc9gV+NLaN0cDzhDvD1s7Y6FZ8RpM= +github.com/pseudomuto/protokit v0.2.0/go.mod h1:2PdH30hxVHsup8KpBTOXTBeMVhJZVio3Q8ViKSAXT0Q= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= @@ -113,11 +132,13 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/topolvm/topolvm v0.25.0 h1:sBWDByfjmsz7IE+4TNZ1hhkSvpDNCbJm0esP8KI5bfE= github.com/topolvm/topolvm v0.25.0/go.mod h1:FE7IJ3t60t+W1b7YPzBlT4qBjlDnMpbVkUZgIRGuyew= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= @@ -126,11 +147,13 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -162,6 +185,7 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= +golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -171,9 +195,12 @@ gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuB google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 h1:L6iMMGrtzgHsWofoFcihmDEMYeDR9KN/ThbPWGrh++g= +google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8= google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e h1:z3vDksarJxsAKM5dmEGv0GHwE2hKJ096wZra71Vs4sw= +google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0 h1:TLkBREm4nIsEcexnCjgQd5GQWaHcqMzwQV0TX9pq8S0= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0/go.mod h1:DNq5QpG7LJqD2AamLZ7zvKE0DEpVl2BSEVjFycAAjRY= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= @@ -209,6 +236,7 @@ k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt sigs.k8s.io/controller-runtime v0.17.0 h1:fjJQf8Ukya+VjogLO6/bNX9HE6Y2xpsO5+fyS26ur/s= sigs.k8s.io/controller-runtime v0.17.0/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= sigs.k8s.io/controller-tools v0.12.1 h1:GyQqxzH5wksa4n3YDIJdJJOopztR5VDM+7qsyg5yE4U= +sigs.k8s.io/controller-tools v0.12.1/go.mod h1:rXlpTfFHZMpZA8aGq9ejArgZiieHd+fkk/fTatY8A2M= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= diff --git a/images/sds-lvm-scheduler-extender/pkg/api/v1alpha1/lvm_volume_group.go b/images/sds-lvm-scheduler-extender/pkg/api/v1alpha1/lvm_volume_group.go new file mode 100644 index 00000000..5ea01488 --- /dev/null +++ b/images/sds-lvm-scheduler-extender/pkg/api/v1alpha1/lvm_volume_group.go @@ -0,0 +1,75 @@ +/* +Copyright 2023 Flant JSC +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type LvmVolumeGroupList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + + Items []LvmVolumeGroup `json:"items"` +} + +type LvmVolumeGroup struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec LvmVolumeGroupSpec `json:"spec"` + Status LvmVolumeGroupStatus `json:"status,omitempty"` +} + +type SpecThinPool struct { + Name string `json:"name"` + Size resource.Quantity `json:"size"` +} + +type LvmVolumeGroupSpec struct { + ActualVGNameOnTheNode string `json:"actualVGNameOnTheNode"` + BlockDeviceNames []string `json:"blockDeviceNames"` + ThinPools []SpecThinPool `json:"thinPools"` + Type string `json:"type"` +} + +type LvmVolumeGroupDevice struct { + BlockDevice string `json:"blockDevice"` + DevSize resource.Quantity `json:"devSize"` + PVSize string `json:"pvSize"` + PVUuid string `json:"pvUUID"` + Path string `json:"path"` +} + +type LvmVolumeGroupNode struct { + Devices []LvmVolumeGroupDevice `json:"devices"` + Name string `json:"name"` +} + +type StatusThinPool struct { + Name string `json:"name"` + ActualSize resource.Quantity `json:"actualSize"` + UsedSize string `json:"usedSize"` +} + +type LvmVolumeGroupStatus struct { + AllocatedSize string `json:"allocatedSize"` + Health string `json:"health"` + Message string `json:"message"` + Nodes []LvmVolumeGroupNode `json:"nodes"` + ThinPools []StatusThinPool `json:"thinPools"` + VGSize string `json:"vgSize"` + VGUuid string `json:"vgUUID"` +} diff --git a/images/sds-lvm-scheduler-extender/pkg/api/v1alpha1/register.go b/images/sds-lvm-scheduler-extender/pkg/api/v1alpha1/register.go new file mode 100644 index 00000000..183f8090 --- /dev/null +++ b/images/sds-lvm-scheduler-extender/pkg/api/v1alpha1/register.go @@ -0,0 +1,47 @@ +/* +Copyright 2023 Flant JSC +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +const ( + LVMVolumeGroupKind = "LvmVolumeGroup" + APIGroup = "storage.deckhouse.io" + APIVersion = "v1alpha1" + TypeMediaAPIVersion = APIGroup + "/" + APIVersion +) + +// SchemeGroupVersion is group version used to register these objects +var ( + SchemeGroupVersion = schema.GroupVersion{ + Group: APIGroup, + Version: APIVersion, + } + SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + AddToScheme = SchemeBuilder.AddToScheme +) + +// Adds the list of known types to Scheme. +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &LvmVolumeGroup{}, + &LvmVolumeGroupList{}, + ) + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) + return nil +} diff --git a/images/sds-lvm-scheduler-extender/pkg/api/v1alpha1/zz_generated.deepcopy.go b/images/sds-lvm-scheduler-extender/pkg/api/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 00000000..ede9ba42 --- /dev/null +++ b/images/sds-lvm-scheduler-extender/pkg/api/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,74 @@ +/* +Copyright 2023 Flant JSC +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import "k8s.io/apimachinery/pkg/runtime" + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LvmVolumeGroup) DeepCopyInto(out *LvmVolumeGroup) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EmptyBlockDevice. +func (in *LvmVolumeGroup) DeepCopy() *LvmVolumeGroup { + if in == nil { + return nil + } + out := new(LvmVolumeGroup) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LvmVolumeGroup) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LvmVolumeGroupList) DeepCopyInto(out *LvmVolumeGroupList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]LvmVolumeGroup, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuestbookList. +func (in *LvmVolumeGroupList) DeepCopy() *LvmVolumeGroupList { + if in == nil { + return nil + } + out := new(LvmVolumeGroupList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LvmVolumeGroupList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} diff --git a/images/sds-lvm-scheduler-extender/pkg/kubutils/kubernetes.go b/images/sds-lvm-scheduler-extender/pkg/kubutils/kubernetes.go new file mode 100644 index 00000000..5bb37ea1 --- /dev/null +++ b/images/sds-lvm-scheduler-extender/pkg/kubutils/kubernetes.go @@ -0,0 +1,35 @@ +/* +Copyright 2023 Flant JSC +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubutils + +import ( + "fmt" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +func KubernetesDefaultConfigCreate() (*rest.Config, error) { + //todo validate empty + clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + clientcmd.NewDefaultClientConfigLoadingRules(), + &clientcmd.ConfigOverrides{}, + ) + + // Get a config to talk to API server + config, err := clientConfig.ClientConfig() + if err != nil { + return nil, fmt.Errorf("config kubernetes error %w", err) + } + return config, nil +} diff --git a/images/sds-lvm-scheduler-extender/pkg/logger/logger.go b/images/sds-lvm-scheduler-extender/pkg/logger/logger.go new file mode 100644 index 00000000..236ada4f --- /dev/null +++ b/images/sds-lvm-scheduler-extender/pkg/logger/logger.go @@ -0,0 +1,81 @@ +/* +Copyright 2023 Flant JSC +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logger + +import ( + "flag" + "fmt" + "github.com/go-logr/logr" + "k8s.io/klog/v2" + "k8s.io/klog/v2/klogr" +) + +const ( + ErrorLevel Verbosity = "0" + WarningLevel Verbosity = "1" + InfoLevel Verbosity = "2" + DebugLevel Verbosity = "3" + TraceLevel Verbosity = "4" +) + +const ( + warnLvl = iota + 1 + infoLvl + debugLvl + traceLvl +) + +type ( + Verbosity string +) + +type Logger struct { + log logr.Logger +} + +func NewLogger(level Verbosity) (*Logger, error) { + klog.InitFlags(nil) + if err := flag.Set("v", string(level)); err != nil { + return nil, err + } + flag.Parse() + + log := klogr.New().WithCallDepth(1) + + return &Logger{log: log}, nil +} + +func (l Logger) GetLogger() logr.Logger { + return l.log +} + +func (l Logger) Error(err error, message string, keysAndValues ...interface{}) { + l.log.Error(err, fmt.Sprintf("ERROR %s", message), keysAndValues...) +} + +func (l Logger) Warning(message string, keysAndValues ...interface{}) { + l.log.V(warnLvl).Info(fmt.Sprintf("WARNING %s", message), keysAndValues...) +} + +func (l Logger) Info(message string, keysAndValues ...interface{}) { + l.log.V(infoLvl).Info(fmt.Sprintf("INFO %s", message), keysAndValues...) +} + +func (l Logger) Debug(message string, keysAndValues ...interface{}) { + l.log.V(debugLvl).Info(fmt.Sprintf("DEBUG %s", message), keysAndValues...) +} + +func (l Logger) Trace(message string, keysAndValues ...interface{}) { + l.log.V(traceLvl).Info(fmt.Sprintf("TRACE %s", message), keysAndValues...) +} diff --git a/images/sds-lvm-scheduler-extender/pkg/scheduler/filter.go b/images/sds-lvm-scheduler-extender/pkg/scheduler/filter.go new file mode 100644 index 00000000..f5219741 --- /dev/null +++ b/images/sds-lvm-scheduler-extender/pkg/scheduler/filter.go @@ -0,0 +1,232 @@ +package scheduler + +import ( + "context" + "encoding/json" + "fmt" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/resource" + "net/http" + "sds-lvm-scheduler-extender/pkg/api/v1alpha1" + "sds-lvm-scheduler-extender/pkg/logger" + "sigs.k8s.io/controller-runtime/pkg/client" + "sync" +) + +const ( + lvmTypeParamKey = "local-lvm.csi.storage.deckhouse.io/lvm-type" + thick = "Thick" + thin = "Thin" +) + +func (s scheduler) filter(w http.ResponseWriter, r *http.Request) { + s.log.Debug("[filter] starts the serving") + var input ExtenderArgs + reader := http.MaxBytesReader(w, r.Body, 10<<20) + err := json.NewDecoder(reader).Decode(&input) + if err != nil || input.Nodes == nil || input.Pod == nil { + s.log.Error(err, "[filter] unable to decode a request") + http.Error(w, "bad request", http.StatusBadRequest) + return + } + + s.log.Debug("[filter] starts to extract requested size") + requested, err := extractRequestedSize(s.client, s.log, input.Pod) + if err != nil { + s.log.Error(err, fmt.Sprintf("[filter] unable to extract request size for a pod %s", input.Pod.Name)) + http.Error(w, "bad request", http.StatusBadRequest) + } + s.log.Debug("[filter] successfully extracted the requested size") + + s.log.Debug("[filter] starts to filter requested nodes") + result, err := filterNodes(s.client, s.log, *input.Nodes, requested) + if err != nil { + s.log.Error(err, "[filter] unable to filter requested nodes") + http.Error(w, "bad request", http.StatusBadRequest) + } + s.log.Debug("[filter] successfully filtered the requested nodes") + + w.Header().Set("content-type", "application/json") + err = json.NewEncoder(w).Encode(result) + if err != nil { + s.log.Error(err, "[filter] unable to encode a response") + http.Error(w, "internal error", http.StatusInternalServerError) + } + s.log.Debug("[filter] ends the serving") +} + +func extractRequestedSize(cl client.Client, log logger.Logger, pod *corev1.Pod) (map[string]int64, error) { + ctx := context.Background() + usedPvc := make([]string, 0, len(pod.Spec.Volumes)) + for _, v := range pod.Spec.Volumes { + if v.PersistentVolumeClaim != nil { + usedPvc = append(usedPvc, v.PersistentVolumeClaim.ClaimName) + } + } + + pvcs := &corev1.PersistentVolumeClaimList{} + err := cl.List(ctx, pvcs) + if err != nil { + return nil, err + } + + pvcMap := make(map[string]corev1.PersistentVolumeClaim, len(pvcs.Items)) + for _, pvc := range pvcs.Items { + pvcMap[pvc.Name] = pvc + } + + scs := &v1.StorageClassList{} + err = cl.List(ctx, scs) + if err != nil { + return nil, err + } + + scMap := make(map[string]v1.StorageClass, len(scs.Items)) + for _, sc := range scs.Items { + scMap[sc.Name] = sc + } + + result := make(map[string]int64, 2) + + for _, pvName := range usedPvc { + pv := pvcMap[pvName] + + scName := pv.Spec.StorageClassName + sc := scMap[*scName] + log.Trace(fmt.Sprintf("[extractRequestedSize] StorageClass %s has LVMType %s", sc.Name, sc.Parameters[lvmTypeParamKey])) + switch sc.Parameters[lvmTypeParamKey] { + case thick: + result[thick] += pv.Spec.Resources.Requests.Storage().Value() + case thin: + result[thin] += pv.Spec.Resources.Requests.Storage().Value() + } + } + + for t, s := range result { + log.Trace(fmt.Sprintf("[extractRequestedSize] pod %s has requested type: %s, size: %d", pod.Name, t, s)) + } + + return result, nil +} + +func filterNodes(cl client.Client, log logger.Logger, nodes corev1.NodeList, requested map[string]int64) (*ExtenderFilterResult, error) { + if len(requested) == 0 { + return &ExtenderFilterResult{ + Nodes: &nodes, + }, nil + } + + ctx := context.Background() + lvgl := &v1alpha1.LvmVolumeGroupList{} + err := cl.List(ctx, lvgl) + if err != nil { + return nil, err + } + + lvgByNodes := make(map[string][]v1alpha1.LvmVolumeGroup, len(lvgl.Items)) + for _, lvg := range lvgl.Items { + for _, node := range lvg.Status.Nodes { + lvgByNodes[node.Name] = append(lvgByNodes[node.Name], lvg) + } + } + + log.Trace(fmt.Sprintf("[filterNodes] sorted LVG by nodes: %+v", lvgByNodes)) + + result := &ExtenderFilterResult{ + Nodes: &corev1.NodeList{}, + FailedNodes: FailedNodesMap{}, + } + + wg := &sync.WaitGroup{} + wg.Add(len(nodes.Items)) + + for _, node := range nodes.Items { + go func(node corev1.Node) { + defer wg.Done() + + lvgs := lvgByNodes[node.Name] + freeSpace, err := getNodeFreeSpace(lvgs) + if err != nil { + log.Error(err, fmt.Sprintf("[filterNodes] unable to get node free space, node: %s, lvgs: %+v", node.Name, lvgs)) + result.FailedNodes[node.Name] = "error occurred while counting free space" + return + } + if freeSpace[thick] < requested[thick] || + freeSpace[thin] < requested[thin] { + result.FailedNodes[node.Name] = "not enough space" + return + } + + result.Nodes.Items = append(result.Nodes.Items, node) + }(node) + } + wg.Wait() + + for _, node := range result.Nodes.Items { + log.Trace(fmt.Sprintf("[filterNodes] suitable node: %s", node.Name)) + } + + for node, reason := range result.FailedNodes { + log.Trace(fmt.Sprintf("[filterNodes] failed node: %s, reason: %s", node, reason)) + } + + return result, nil +} + +func getNodeFreeSpace(lvgs []v1alpha1.LvmVolumeGroup) (map[string]int64, error) { + freeSpaces := make(map[string]int64, 2) + + for _, lvg := range lvgs { + // здесь не нужно делать выборку по типу, мы просто смотрим, сколько есть места такого и такого (а не одно из двух) + + // выбираю максимальное свободное место из thin pool + for _, tp := range lvg.Status.ThinPools { + thinSpace, err := getThinPoolFreeSpace(tp) + if err != nil { + return nil, err + } + + if freeSpaces[thin] < thinSpace.Value() { + freeSpaces[thin] = thinSpace.Value() + } + } + + thickSpace, err := getVGFreeSpace(&lvg) + if err != nil { + return nil, err + } + + if freeSpaces[thick] < thickSpace.Value() { + freeSpaces[thick] = thickSpace.Value() + } + } + + return freeSpaces, nil +} + +func getVGFreeSpace(lvg *v1alpha1.LvmVolumeGroup) (resource.Quantity, error) { + free, err := resource.ParseQuantity(lvg.Status.VGSize) + if err != nil { + return resource.Quantity{}, err + } + + used, err := resource.ParseQuantity(lvg.Status.AllocatedSize) + if err != nil { + return resource.Quantity{}, err + } + + free.Sub(used) + return free, nil +} + +func getThinPoolFreeSpace(tp v1alpha1.StatusThinPool) (resource.Quantity, error) { + free := tp.ActualSize + used, err := resource.ParseQuantity(tp.UsedSize) + if err != nil { + return resource.Quantity{}, err + } + free.Sub(used) + + return free, nil +} diff --git a/images/sds-lvm-scheduler-extender/pkg/scheduler/predicate_test.go b/images/sds-lvm-scheduler-extender/pkg/scheduler/filter_test.go similarity index 100% rename from images/sds-lvm-scheduler-extender/pkg/scheduler/predicate_test.go rename to images/sds-lvm-scheduler-extender/pkg/scheduler/filter_test.go diff --git a/images/sds-lvm-scheduler-extender/pkg/scheduler/predicate.go b/images/sds-lvm-scheduler-extender/pkg/scheduler/predicate.go deleted file mode 100644 index 29ed6eff..00000000 --- a/images/sds-lvm-scheduler-extender/pkg/scheduler/predicate.go +++ /dev/null @@ -1,95 +0,0 @@ -package scheduler - -import ( - "encoding/json" - "net/http" - "strconv" - "strings" - "sync" - - "github.com/topolvm/topolvm" - corev1 "k8s.io/api/core/v1" -) - -func filterNodes(nodes corev1.NodeList, requested map[string]int64) ExtenderFilterResult { - if len(requested) == 0 { - return ExtenderFilterResult{ - Nodes: &nodes, - } - } - - failedNodes := make([]string, len(nodes.Items)) - wg := &sync.WaitGroup{} - wg.Add(len(nodes.Items)) - for i := range nodes.Items { - reason := &failedNodes[i] - node := nodes.Items[i] - go func() { - *reason = filterNode(node, requested) - wg.Done() - }() - } - wg.Wait() - result := ExtenderFilterResult{ - Nodes: &corev1.NodeList{}, - FailedNodes: FailedNodesMap{}, - } - for i, reason := range failedNodes { - if len(reason) == 0 { - result.Nodes.Items = append(result.Nodes.Items, nodes.Items[i]) - } else { - result.FailedNodes[nodes.Items[i].Name] = reason - } - } - return result -} - -func filterNode(node corev1.Node, requested map[string]int64) string { - for dc, required := range requested { - val, ok := node.Annotations[topolvm.GetCapacityKeyPrefix()+dc] - if !ok { - return "no capacity annotation" - } - capacity, err := strconv.ParseUint(val, 10, 64) - if err != nil { - return "bad capacity annotation: " + val - } - if capacity < uint64(required) { - return "out of VG free space" - } - } - return "" -} - -func extractRequestedSize(pod *corev1.Pod) map[string]int64 { - result := make(map[string]int64) - for k, v := range pod.Annotations { - if !strings.HasPrefix(k, topolvm.GetCapacityKeyPrefix()) { - continue - } - dc := k[len(topolvm.GetCapacityKeyPrefix()):] - capacity, err := strconv.ParseInt(v, 10, 64) - if err != nil { - continue - } - result[dc] = capacity - } - - return result -} - -func (s scheduler) predicate(w http.ResponseWriter, r *http.Request) { - var input ExtenderArgs - - reader := http.MaxBytesReader(w, r.Body, 10<<20) - err := json.NewDecoder(reader).Decode(&input) - if err != nil || input.Nodes == nil || input.Pod == nil { - http.Error(w, "bad request", http.StatusBadRequest) - return - } - - requested := extractRequestedSize(input.Pod) - result := filterNodes(*input.Nodes, requested) - w.Header().Set("content-type", "application/json") - json.NewEncoder(w).Encode(result) -} diff --git a/images/sds-lvm-scheduler-extender/pkg/scheduler/prioritize.go b/images/sds-lvm-scheduler-extender/pkg/scheduler/prioritize.go index 95334480..50f109d3 100644 --- a/images/sds-lvm-scheduler-extender/pkg/scheduler/prioritize.go +++ b/images/sds-lvm-scheduler-extender/pkg/scheduler/prioritize.go @@ -1,106 +1,115 @@ package scheduler import ( + "context" "encoding/json" + "fmt" + corev1 "k8s.io/api/core/v1" "math" "net/http" - "strconv" - "strings" + "sds-lvm-scheduler-extender/pkg/api/v1alpha1" + "sds-lvm-scheduler-extender/pkg/logger" + "sigs.k8s.io/controller-runtime/pkg/client" "sync" - - "github.com/topolvm/topolvm" - corev1 "k8s.io/api/core/v1" ) -func capacityToScore(capacity uint64, divisor float64) int { - gb := capacity >> 30 - - // Avoid logarithm of zero, which diverges to negative infinity. - if gb == 0 { - // If there is a non-nil capacity but we dont have at least one gigabyte, we score it with one. - // This is because the capacityToScore precision is at the gigabyte level. - // TODO: introduce another scheduling algorithm for byte-level precision. - if capacity > 0 { - return 1 - } +func (s scheduler) prioritize(w http.ResponseWriter, r *http.Request) { + s.log.Debug("[prioritize] starts serving") + var input ExtenderArgs + reader := http.MaxBytesReader(w, r.Body, 10<<20) + err := json.NewDecoder(reader).Decode(&input) + if err != nil { + s.log.Error(err, "[prioritize] unable to decode a request") + http.Error(w, "Bad Request.", http.StatusBadRequest) + return + } - return 0 + result, err := scoreNodes(s.client, s.log, input.Nodes.Items, s.defaultDivisor) + if err != nil { + s.log.Error(err, "[prioritize] unable to score nodes") + http.Error(w, "Bad Request.", http.StatusBadRequest) + return } - converted := int(math.Log2(float64(gb) / divisor)) - switch { - case converted < 1: - return 1 - case converted > 10: - return 10 - default: - return converted + w.Header().Set("content-type", "application/json") + err = json.NewEncoder(w).Encode(result) + if err != nil { + s.log.Error(err, "[prioritize] unable to encode a response") + http.Error(w, "internal error", http.StatusInternalServerError) } + s.log.Debug("[prioritize] ends serving") } -func scoreNodes(pod *corev1.Pod, nodes []corev1.Node, defaultDivisor float64, divisors map[string]float64) []HostPriority { - var dcs []string - for k := range pod.Annotations { - if strings.HasPrefix(k, topolvm.GetCapacityKeyPrefix()) { - dcs = append(dcs, k[len(topolvm.GetCapacityKeyPrefix()):]) - } +func scoreNodes(cl client.Client, log logger.Logger, nodes []corev1.Node, divisor float64) ([]HostPriority, error) { + ctx := context.Background() + lvgl := &v1alpha1.LvmVolumeGroupList{} + err := cl.List(ctx, lvgl) + if err != nil { + return nil, err } - if len(dcs) == 0 { - return nil + + lvgByNodes := make(map[string][]v1alpha1.LvmVolumeGroup, len(lvgl.Items)) + for _, lvg := range lvgl.Items { + for _, node := range lvg.Status.Nodes { + lvgByNodes[node.Name] = append(lvgByNodes[node.Name], lvg) + } } - result := make([]HostPriority, len(nodes)) + log.Trace(fmt.Sprintf("[scoreNodes] sorted LVG by nodes: %+v", lvgByNodes)) + + result := make([]HostPriority, 0, len(nodes)) + + // TODO: probably should score the nodes exactly to their free space wg := &sync.WaitGroup{} wg.Add(len(nodes)) - for i := range nodes { - r := &result[i] - item := nodes[i] - go func() { - score := scoreNode(item, dcs, defaultDivisor, divisors) - *r = HostPriority{Host: item.Name, Score: score} - wg.Done() - }() - } - wg.Wait() - return result -} + for _, node := range nodes { + go func(node corev1.Node) { + defer wg.Done() -func scoreNode(item corev1.Node, deviceClasses []string, defaultDivisor float64, divisors map[string]float64) int { - minScore := math.MaxInt32 - for _, dc := range deviceClasses { - if val, ok := item.Annotations[topolvm.GetCapacityKeyPrefix()+dc]; ok { - capacity, _ := strconv.ParseUint(val, 10, 64) - var divisor float64 - if v, ok := divisors[dc]; ok { - divisor = v - } else { - divisor = defaultDivisor - } - score := capacityToScore(capacity, divisor) - if score < minScore { - minScore = score + lvgs := lvgByNodes[node.Name] + freeSpace, err := getNodeFreeSpace(lvgs) + if err != nil { + log.Error(err, fmt.Sprintf("[scoreNodes] unable to get node free space, node: %s, lvgs: %+v", node.Name, lvgs)) + return } - } + + score := getNodeScore(freeSpace, divisor) + result = append(result, HostPriority{Host: node.Name, Score: score}) + }(node) } - if minScore == math.MaxInt32 { - minScore = 0 + wg.Wait() + + for _, n := range result { + log.Trace(fmt.Sprintf("[scoreNodes] host: %s", n.Host)) + log.Trace(fmt.Sprintf("[scoreNodes] score: %d", n.Score)) } - return minScore + + return result, nil } -func (s scheduler) prioritize(w http.ResponseWriter, r *http.Request) { - var input ExtenderArgs +func getNodeScore(freeSpace map[string]int64, divisor float64) int { + capacity := freeSpace[thin] + freeSpace[thick] + gb := capacity >> 30 - reader := http.MaxBytesReader(w, r.Body, 10<<20) - err := json.NewDecoder(reader).Decode(&input) - if err != nil { - http.Error(w, "Bad Request.", http.StatusBadRequest) - return - } + // Avoid logarithm of zero, which diverges to negative infinity. + if gb == 0 { + // If there is a non-nil capacity, but we don't have at least one gigabyte, we score it with one. + // This is because the capacityToScore precision is at the gigabyte level. + if capacity > 0 { + return 1 + } - result := scoreNodes(input.Pod, input.Nodes.Items, s.defaultDivisor, s.divisors) + return 0 + } - w.Header().Set("content-type", "application/json") - json.NewEncoder(w).Encode(result) + converted := int(math.Log2(float64(gb) / divisor)) + switch { + case converted < 1: + return 1 + case converted > 10: + return 10 + default: + return converted + } } diff --git a/images/sds-lvm-scheduler-extender/pkg/scheduler/route.go b/images/sds-lvm-scheduler-extender/pkg/scheduler/route.go index 7a3f6cb3..d99ddf54 100644 --- a/images/sds-lvm-scheduler-extender/pkg/scheduler/route.go +++ b/images/sds-lvm-scheduler-extender/pkg/scheduler/route.go @@ -3,37 +3,45 @@ package scheduler import ( "fmt" "net/http" + "sds-lvm-scheduler-extender/pkg/logger" + + "sigs.k8s.io/controller-runtime/pkg/client" ) type scheduler struct { defaultDivisor float64 - divisors map[string]float64 + log logger.Logger + client client.Client } func (s scheduler) ServeHTTP(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { - case "/predicate": - s.predicate(w, r) + case "/filter": + s.log.Debug("[ServeHTTP] filter route starts handling the request") + s.filter(w, r) + s.log.Debug("[ServeHTTP] filter route ends handling the request") case "/prioritize": + s.log.Debug("[ServeHTTP] prioritize route starts handling the request") s.prioritize(w, r) + s.log.Debug("[ServeHTTP] prioritize route ends handling the request") case "/status": + s.log.Debug("[ServeHTTP] status route starts handling the request") status(w, r) + s.log.Debug("[ServeHTTP] status route ends handling the request") default: http.Error(w, "not found", http.StatusNotFound) } } // NewHandler return new http.Handler of the scheduler extender -func NewHandler(defaultDiv float64, divisors map[string]float64) (http.Handler, error) { - for _, divisor := range divisors { - if divisor <= 0 { - return nil, fmt.Errorf("invalid divisor: %f", divisor) - } - } - return scheduler{defaultDiv, divisors}, nil +func NewHandler(cl client.Client, log logger.Logger, defaultDiv float64) (http.Handler, error) { + return scheduler{defaultDiv, log, cl}, nil } func status(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - w.Write([]byte("ok")) + _, err := w.Write([]byte("ok")) + if err != nil { + fmt.Println(fmt.Sprintf("error occurs on status route, err: %s", err.Error())) + } } diff --git a/templates/registry-secret.yaml b/templates/registry-secret.yaml index 1e6eddc4..655fe216 100644 --- a/templates/registry-secret.yaml +++ b/templates/registry-secret.yaml @@ -7,8 +7,8 @@ metadata: {{- include "helm_lib_module_labels" (list .) | nindent 2 }} type: kubernetes.io/dockerconfigjson data: -{{- if dig "registry" "dockercfg" false .Values.sdsLvmProvisioner }} - .dockerconfigjson: {{ .Values.sdsLvmProvisioner.registry.dockercfg }} +{{- if dig "registry" "dockercfg" false .Values.sdsLvm }} + .dockerconfigjson: {{ .Values.sdsLvm.registry.dockercfg }} {{- else }} .dockerconfigjson: "eyJhdXRocyI6IHsgInJlZ2lzdHJ5LmRlY2tob3VzZS5pbyI6IHt9fX0=" {{- end }} diff --git a/templates/sds-lvm-scheduler/configmap.yaml b/templates/sds-lvm-scheduler/configmap.yaml index 115cee4c..6bfa6e97 100644 --- a/templates/sds-lvm-scheduler/configmap.yaml +++ b/templates/sds-lvm-scheduler/configmap.yaml @@ -9,7 +9,8 @@ data: scheduler-extender-config.yaml: |- listen: "localhost:8099" default-divisor: 1 - {{- if semverCompare ">= 1.22" .Values.global.discovery.kubernetesVersion }} + log-level: "2" + {{- if semverCompare ">= 1.22" .Values.global.discovery.kubernetesVersion }} scheduler-config.yaml: |- {{- if semverCompare ">= 1.23" .Values.global.discovery.kubernetesVersion }} apiVersion: kubescheduler.config.k8s.io/v1beta3 diff --git a/templates/sds-lvm-scheduler/deployment.yaml b/templates/sds-lvm-scheduler/deployment.yaml index 1c54df32..49b7b0df 100644 --- a/templates/sds-lvm-scheduler/deployment.yaml +++ b/templates/sds-lvm-scheduler/deployment.yaml @@ -135,6 +135,7 @@ spec: image: {{ include "helm_lib_module_image" (list . "sdsLvmSchedulerExtender") }} imagePullPolicy: IfNotPresent args: + - sds-lvm-scheduler-extender - --config=/etc/sds-lvm-scheduler-extender/scheduler-extender-config.yaml volumeMounts: - mountPath: /etc/sds-lvm-scheduler-extender diff --git a/templates/sds-lvm-scheduler/rbac-for-us.yaml b/templates/sds-lvm-scheduler/rbac-for-us.yaml index 9be58851..9b3da75e 100644 --- a/templates/sds-lvm-scheduler/rbac-for-us.yaml +++ b/templates/sds-lvm-scheduler/rbac-for-us.yaml @@ -35,7 +35,7 @@ subjects: namespace: d8-{{ .Chart.Name }} --- apiVersion: rbac.authorization.k8s.io/v1 -kind: Role +kind: ClusterRole metadata: name: sds-lvm-scheduler namespace: d8-{{ .Chart.Name }} @@ -44,16 +44,19 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["create", "get", "update"] + - apiGroups: [ "storage.deckhouse.io" ] + resources: [ "lvmvolumegroups" ] + verbs: [ "list" ] --- apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding +kind: ClusterRoleBinding metadata: name: sds-lvm-scheduler namespace: d8-{{ .Chart.Name }} {{- include "helm_lib_module_labels" (list . (dict "app" "sds-lvm-scheduler")) | nindent 2 }} roleRef: apiGroup: rbac.authorization.k8s.io - kind: Role + kind: ClusterRole name: sds-lvm-scheduler subjects: - kind: ServiceAccount