From d3c3baba81defbc6774fe3b8b95bfd189d596d69 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Thu, 12 Mar 2020 16:35:15 -0700 Subject: [PATCH] QoS Sort plugin sample src code --- .gitignore | 19 +++++ cmd/main.go | 39 +++++++++ pkg/qos/README.md | 23 +++++ pkg/qos/queuesort.go | 63 ++++++++++++++ pkg/qos/queuesort_test.go | 155 ++++++++++++++++++++++++++++++++++ test/integration/main_test.go | 27 ++++++ test/integration/qos_test.go | 136 +++++++++++++++++++++++++++++ test/util/pod.go | 36 ++++++++ test/util/scheduler.go | 56 ++++++++++++ 9 files changed, 554 insertions(+) create mode 100644 .gitignore create mode 100644 cmd/main.go create mode 100644 pkg/qos/README.md create mode 100644 pkg/qos/queuesort.go create mode 100644 pkg/qos/queuesort_test.go create mode 100644 test/integration/main_test.go create mode 100644 test/integration/qos_test.go create mode 100644 test/util/pod.go create mode 100644 test/util/scheduler.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..0e076b53e --- /dev/null +++ b/.gitignore @@ -0,0 +1,19 @@ +# build and test outputs +/bin/ +/_output/ +/_artifacts/ + +# used for the code generators only +/vendor/ + +# macOS +.DS_Store + +# files generated by editors +.idea/ +*.iml +.vscode/ +*.swp +*.sublime-project +*.sublime-workspace +*~ \ No newline at end of file diff --git a/cmd/main.go b/cmd/main.go new file mode 100644 index 000000000..8aeec6465 --- /dev/null +++ b/cmd/main.go @@ -0,0 +1,39 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "math/rand" + "os" + "time" + + "k8s.io/kubernetes/cmd/kube-scheduler/app" + "sigs.k8s.io/scheduler-plugins/pkg/qos" +) + +func main() { + rand.Seed(time.Now().UnixNano()) + // Register custom plugins to the scheduler framework. + // Later they can consist of scheduler profile(s) and hence + // used by various kinds of workloads. + command := app.NewSchedulerCommand( + app.WithPlugin(qos.Name, qos.New), + ) + if err := command.Execute(); err != nil { + os.Exit(1) + } +} diff --git a/pkg/qos/README.md b/pkg/qos/README.md new file mode 100644 index 000000000..76c204f78 --- /dev/null +++ b/pkg/qos/README.md @@ -0,0 +1,23 @@ +# Overview + +This folder holds some sample plugin implementations based on [QoS +(Quality of Service) class](https://kubernetes.io/docs/tasks/configure-pod-container/quality-service-pod/) +of Pods. + +## Maturity Level + + + +- [x] 💡 Sample (for demonstrating and inspiring purpose) +- [ ] 👶 Alpha (used in companies for pilot projects) +- [ ] 👦 Beta (used in companies and developed actively) +- [ ] 👨 Stable (used in companies for production workloads) + +## QOS QueueSort Plugin + +Sorts pods by .spec.priority and breaks ties by the [quality of service class](https://kubernetes.io/docs/tasks/configure-pod-container/quality-service-pod/#qos-classes). +Specifically, this plugin enqueue the Pods with the following order: + +- Guaranteed (requests == limits) +- Burstable (requests < limits) +- BestEffort (requests and limits not set) diff --git a/pkg/qos/queuesort.go b/pkg/qos/queuesort.go new file mode 100644 index 000000000..a6d64b364 --- /dev/null +++ b/pkg/qos/queuesort.go @@ -0,0 +1,63 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package qos + +import ( + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/kubernetes/pkg/api/v1/pod" + v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos" + framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" +) + +// Name is the name of the plugin used in the plugin registry and configurations. +const Name = "QoSSort" + +// QoSSort is a plugin that implements QoS class based sorting. +type Sort struct{} + +var _ framework.QueueSortPlugin = &Sort{} + +// Name returns name of the plugin. +func (pl *Sort) Name() string { + return Name +} + +// Less is the function used by the activeQ heap algorithm to sort pods. +// It sorts pods based on their priority. When priorities are equal, it uses +// PodInfo.timestamp. +func (*Sort) Less(pInfo1, pInfo2 *framework.PodInfo) bool { + p1 := pod.GetPodPriority(pInfo1.Pod) + p2 := pod.GetPodPriority(pInfo2.Pod) + return (p1 > p2) || (p1 == p2 && compQOS(pInfo1.Pod, pInfo2.Pod)) +} + +func compQOS(p1, p2 *v1.Pod) bool { + p1QOS, p2QOS := v1qos.GetPodQOS(p1), v1qos.GetPodQOS(p2) + if p1QOS == v1.PodQOSGuaranteed { + return true + } else if p1QOS == v1.PodQOSBurstable { + return p2QOS != v1.PodQOSGuaranteed + } else { + return p2QOS == v1.PodQOSBestEffort + } +} + +// New initializes a new plugin and returns it. +func New(_ *runtime.Unknown, _ framework.FrameworkHandle) (framework.Plugin, error) { + return &Sort{}, nil +} \ No newline at end of file diff --git a/pkg/qos/queuesort_test.go b/pkg/qos/queuesort_test.go new file mode 100644 index 000000000..993e56d87 --- /dev/null +++ b/pkg/qos/queuesort_test.go @@ -0,0 +1,155 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package qos + +import ( + "testing" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" +) + +func TestSortLess(t *testing.T) { + tests := []struct { + name string + pInfo1 *framework.PodInfo + pInfo2 *framework.PodInfo + want bool + }{ + { + name: "p1's priority greater than p2", + pInfo1: &framework.PodInfo{ + Pod: makePod("p1", 100, nil, nil), + }, + pInfo2: &framework.PodInfo{ + Pod: makePod("p2", 50, nil, nil), + }, + want: true, + }, + { + name: "p1's priority less than p2", + pInfo1: &framework.PodInfo{ + Pod: makePod("p1", 50, nil, nil), + }, + pInfo2: &framework.PodInfo{ + Pod: makePod("p2", 80, nil, nil), + }, + want: false, + }, + { + name: "p1 and p2 are both BestEfforts", + pInfo1: &framework.PodInfo{ + Pod: makePod("p1", 0, nil, nil), + }, + pInfo2: &framework.PodInfo{ + Pod: makePod("p2", 0, nil, nil), + }, + want: true, + }, + { + name: "p1 is BestEfforts, p2 is Guaranteed", + pInfo1: &framework.PodInfo{ + Pod: makePod("p1", 0, nil, nil), + }, + pInfo2: &framework.PodInfo{ + Pod: makePod("p2", 0, getResList("100m", "100Mi"), getResList("100m", "100Mi")), + }, + want: false, + }, + { + name: "p1 is Burstable, p2 is Guaranteed", + pInfo1: &framework.PodInfo{ + Pod: makePod("p1", 0, getResList("100m", "100Mi"), getResList("200m", "200Mi")), + }, + pInfo2: &framework.PodInfo{ + Pod: makePod("p2", 0, getResList("100m", "100Mi"), getResList("100m", "100Mi")), + }, + want: false, + }, + { + name: "both p1 and p2 are Burstable", + pInfo1: &framework.PodInfo{ + Pod: makePod("p1", 0, getResList("100m", "100Mi"), getResList("200m", "200Mi")), + }, + pInfo2: &framework.PodInfo{ + Pod: makePod("p2", 0, getResList("100m", "100Mi"), getResList("200m", "200Mi")), + }, + want: true, + }, + { + name: "p1 is Guaranteed, p2 is Burstable", + pInfo1: &framework.PodInfo{ + Pod: makePod("p1", 0, getResList("100m", "100Mi"), getResList("100m", "100Mi")), + }, + pInfo2: &framework.PodInfo{ + Pod: makePod("p2", 0, getResList("100m", "100Mi"), getResList("200m", "200Mi")), + }, + want: true, + }, + { + name: "both p1 and p2 are Guaranteed", + pInfo1: &framework.PodInfo{ + Pod: makePod("p1", 0, getResList("100m", "100Mi"), getResList("100m", "100Mi")), + }, + pInfo2: &framework.PodInfo{ + Pod: makePod("p2", 0, getResList("100m", "100Mi"), getResList("100m", "100Mi")), + }, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &Sort{} + if got := s.Less(tt.pInfo1, tt.pInfo2); got != tt.want { + t.Errorf("Less() = %v, want %v", got, tt.want) + } + }) + } +} + +func makePod(name string, priority int32, requests, limits v1.ResourceList) *v1.Pod { + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: v1.PodSpec{ + Priority: &priority, + Containers: []v1.Container{ + { + Name: name, + Resources: v1.ResourceRequirements{ + Requests: requests, + Limits: limits, + }, + }, + }, + }, + } +} + +func getResList(cpu, memory string) v1.ResourceList { + res := v1.ResourceList{} + if cpu != "" { + res[v1.ResourceCPU] = resource.MustParse(cpu) + } + if memory != "" { + res[v1.ResourceMemory] = resource.MustParse(memory) + } + return res +} diff --git a/test/integration/main_test.go b/test/integration/main_test.go new file mode 100644 index 000000000..529b6da48 --- /dev/null +++ b/test/integration/main_test.go @@ -0,0 +1,27 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "testing" + + "k8s.io/kubernetes/test/integration/framework" +) + +func TestMain(m *testing.M) { + framework.EtcdMain(m.Run) +} diff --git a/test/integration/qos_test.go b/test/integration/qos_test.go new file mode 100644 index 000000000..e03920885 --- /dev/null +++ b/test/integration/qos_test.go @@ -0,0 +1,136 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "context" + "testing" + "time" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/kubernetes/pkg/scheduler" + schedapi "k8s.io/kubernetes/pkg/scheduler/apis/config" + framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" + st "k8s.io/kubernetes/pkg/scheduler/testing" + testutils "k8s.io/kubernetes/test/integration/util" + imageutils "k8s.io/kubernetes/test/utils/image" + + "sigs.k8s.io/scheduler-plugins/pkg/qos" + "sigs.k8s.io/scheduler-plugins/test/util" +) + +func TestQOSPlugin(t *testing.T) { + registry := framework.Registry{qos.Name: qos.New} + profile := schedapi.KubeSchedulerProfile{ + SchedulerName: v1.DefaultSchedulerName, + Plugins: &schedapi.Plugins{ + QueueSort: &schedapi.PluginSet{ + Enabled: []schedapi.Plugin{ + {Name: qos.Name}, + }, + Disabled: []schedapi.Plugin{ + {Name: "*"}, + }, + }, + }, + } + testCtx := util.InitTestSchedulerWithOptions( + t, + testutils.InitTestMaster(t, "sched-qos", nil), + false, + scheduler.WithProfiles(profile), + scheduler.WithFrameworkOutOfTreeRegistry(registry), + ) + defer testutils.CleanupTest(t, testCtx) + + cs, ns := testCtx.ClientSet, testCtx.NS.Name + // Create a Node. + nodeName := "fake-node" + node := st.MakeNode().Name("fake-node").Label("node", nodeName).Obj() + node.Status.Capacity = v1.ResourceList{ + v1.ResourcePods: *resource.NewQuantity(32, resource.DecimalSI), + v1.ResourceCPU: *resource.NewMilliQuantity(500, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(500, resource.DecimalSI), + } + node, err := cs.CoreV1().Nodes().Create(context.TODO(), node, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create Node %q: %v", nodeName, err) + } + + // Create 3 Pods. + var pods []*v1.Pod + podNames := []string{"bestefforts", "burstable", "guaranteed"} + pause := imageutils.GetPauseImageName() + for i := 0; i < len(podNames); i++ { + pod := st.MakePod().Namespace(ns).Name(podNames[i]).Container(pause).Obj() + pods = append(pods, pod) + } + // Make pods[0] BestEfforts (i.e., do nothing). + // Make pods[1] Burstable. + pods[1].Spec.Containers[0].Resources = v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(100, resource.DecimalSI), + }, + Limits: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(200, resource.DecimalSI), + }, + } + // Make pods[2] Guaranteed. + pods[2].Spec.Containers[0].Resources = v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(100, resource.DecimalSI), + }, + Limits: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(100, resource.DecimalSI), + }, + } + + // Create 3 Pods with the order: BestEfforts, Burstable, Guaranteed. + // We will expect them to be scheduled in a reversed order. + t.Logf("Start to create 3 Pods.") + for i := range pods { + t.Logf("Creating Pod %q", pods[i].Name) + _, err = cs.CoreV1().Pods(ns).Create(context.TODO(), pods[i], metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create Pod %q: %v", pods[i].Name, err) + } + } + + // Wait for all Pods are in the scheduling queue. + err = wait.Poll(time.Millisecond*200, wait.ForeverTestTimeout, func() (bool, error) { + if len(testCtx.Scheduler.SchedulingQueue.PendingPods()) == len(pods) { + return true, nil + } + return false, nil + }) + if err != nil { + t.Fatal(err) + } + + // Expect Pods are popped in the QoS class order. + for i := len(podNames) - 1; i >= 0; i-- { + podInfo := testCtx.Scheduler.NextPod() + if podInfo.Pod.Name != podNames[i] { + t.Errorf("Expect Pod %q, but got %q", podNames[i], podInfo.Pod.Name) + } else { + t.Logf("Pod %q is popped out as expected.", podInfo.Pod.Name) + } + } +} diff --git a/test/util/pod.go b/test/util/pod.go new file mode 100644 index 000000000..bc868384b --- /dev/null +++ b/test/util/pod.go @@ -0,0 +1,36 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "context" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" +) + +func PrintPods(t *testing.T, cs clientset.Interface, ns string) { + podList, err := cs.CoreV1().Pods(ns).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + t.Errorf("Failed to list Pods: %v", err) + return + } + for _, pod := range podList.Items { + t.Logf("Pod %q scheduled to %q", pod.Name, pod.Spec.NodeName) + } +} diff --git a/test/util/scheduler.go b/test/util/scheduler.go new file mode 100644 index 000000000..18ed929e5 --- /dev/null +++ b/test/util/scheduler.go @@ -0,0 +1,56 @@ +package util + +import ( + "testing" + + "k8s.io/client-go/informers" + coreinformers "k8s.io/client-go/informers/core/v1" + "k8s.io/client-go/tools/events" + "k8s.io/kubernetes/pkg/scheduler" + "k8s.io/kubernetes/pkg/scheduler/profile" + testutils "k8s.io/kubernetes/test/integration/util" +) + +// InitTestSchedulerWithOptions initializes a test environment and creates a scheduler with default +// configuration and other options. +// TODO(Huang-Wei): refactor the same function in the upstream, and remove here. +func InitTestSchedulerWithOptions( + t *testing.T, + testCtx *testutils.TestContext, + startScheduler bool, + opts ...scheduler.Option, +) *testutils.TestContext { + testCtx.InformerFactory = informers.NewSharedInformerFactory(testCtx.ClientSet, 0) + + var podInformer coreinformers.PodInformer + + podInformer = testCtx.InformerFactory.Core().V1().Pods() + var err error + eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{ + Interface: testCtx.ClientSet.EventsV1beta1().Events(""), + }) + + testCtx.Scheduler, err = scheduler.New( + testCtx.ClientSet, + testCtx.InformerFactory, + podInformer, + profile.NewRecorderFactory(eventBroadcaster), + testCtx.Ctx.Done(), + opts..., + ) + + if err != nil { + t.Fatalf("Couldn't create scheduler: %v", err) + } + + eventBroadcaster.StartRecordingToSink(testCtx.Ctx.Done()) + + testCtx.InformerFactory.Start(testCtx.Scheduler.StopEverything) + testCtx.InformerFactory.WaitForCacheSync(testCtx.Scheduler.StopEverything) + + if startScheduler { + go testCtx.Scheduler.Run(testCtx.Ctx) + } + + return testCtx +}